maltese 0.9.1 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c8f0fb880fee42bab612c9a17aee2ac8c19bb5c1c320a87ed6b5efb3804b581
-  data.tar.gz: 4825f893e0fcc9355e5c72b0597f73d471fc813491b1fd778e161633e7665815
+  metadata.gz: 4b25d7aed1fa4b41a642a8bf8449e067c67fa28861cf319c54d24b4ffccbcd6d
+  data.tar.gz: af043d259c92a902d5a42c281a0746fc2cf65f356ebb2bec161cc707f89f9b3a
 SHA512:
-  metadata.gz: c30bd244311af98acd5e3b23353f9336a9abdce721fb730496e1e16e239dc8fc07173a23669fb98621588d45fa09817c45f671370192c54eb2f16c7b57023743
-  data.tar.gz: c3ad6590a4ec6a16e245db103d5e9dcd0198284c282db98b848f47e7dae16bddc04c435a0e83db789049c5e13ad427f10aae703b3c6dbe597f29ff4b27f98b4f
+  metadata.gz: e1399ede091a478151153b5071b7881ae1bedd9068ac017ab34c1cfcf1931559a607f0f2ccba83b1018b3a251898e4498d8557267905478c7dd48e322b288722
+  data.tar.gz: 50cd6f3a1ee8a4b0c508805720efcc58db31208dd7842668b6785cf0684bd03059d2872113e002d4c89c5e501fb2924f6323ff24caa3db5227c86517fbbef7c3

data/Dockerfile

@@ -11,6 +11,6 @@ RUN apt-get update && apt-get upgrade -y -o Dpkg::Options::="--force-confold" &&
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Install maltese gem
-RUN /sbin/setuser app gem install maltese -v 0.9.1
+RUN /sbin/setuser app gem install maltese -v 0.9.2
 
 CMD maltese sitemap --sitemap_bucket $SITEMAP_BUCKET --rack_env $RACK_ENV --access_key $AWS_ACCESS_KEY_ID --secret_key $AWS_SECRET_ACCESS_KEY --region $AWS_REGION

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    maltese (0.9.1)
+    maltese (0.9.2)
       activesupport (>= 4.2.5, < 6)
       aws-sdk-s3 (~> 1.19)
       dotenv (~> 2.1, >= 2.1.1)
@@ -9,6 +9,7 @@ PATH
       logstash-logger (~> 0.26.1)
       maremma (~> 4.1)
       mime-types (~> 3.1)
+      retriable (~> 3.1)
       sitemap_generator (~> 6.0)
       thor (~> 0.19)
 
@@ -88,6 +89,7 @@ GEM
     rack-test (0.8.3)
       rack (>= 1.0, < 3)
     rake (12.3.3)
+    retriable (3.1.2)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)

data/lib/maltese/sitemap.rb

@@ -1,6 +1,9 @@
 require 'logstash-logger'
+require 'retriable'
 
 module Maltese
+  class ::BadGatewayError < StandardError; end
+
   class Sitemap
     attr_reader :sitemap_bucket, :rack_env, :access_key, :secret_key, :region, :logger
 
@@ -105,21 +108,35 @@ module Maltese
 
       # walk through paginated results
       while options[:url] do
-        response = get_data(options[:url])
-
-        if response.status == 200
-          link_count = parse_data(response)
-          logger.info "#{link_count} DOIs parsed."
-          options[:url] = response.body.dig("links", "next")
-        else
-          logger.error "An error occured for URL #{options[:url]}."
-          logger.error "Error message: #{response.body.fetch("errors").inspect}" if response.body.fetch("errors", nil).present?
+        begin
+          response = nil
+
+          # retry on temporal errors (status codes 408 and 502)
+          Retriable.retriable(base_interval: 10, multiplier: 2) do
+            response = get_data(options[:url])
+
+            raise Timeout::Error, "A timeout error occured for URL #{options[:url]}." if response.status == 408
+            raise BadGatewayError, "A bad gateway error occured for URL #{options[:url]}." if response.status == 502
+          end
+
+          if response.status == 200
+            link_count = parse_data(response)
+            logger.info "#{link_count} DOIs parsed."
+            options[:url] = response.body.dig("links", "next")
+          else
+            logger.error "An error occured for URL #{options[:url]}."
+            logger.error "Error: #{response.body.fetch("errors").inspect}" if response.body.fetch("errors", nil).present?
+            error_count += 1
+            options[:url] = nil
+          end
+        rescue => exception
+          logger.error "Error: #{exception.message}."
           error_count += 1
           options[:url] = nil
-        end 
-
-        # don't loop when testing
-        break if rack_env == "test"     
+        ensure
+          # don't loop when testing
+          break if rack_env == "test"
+        end  
       end
 
       return link_count if error_count > 0

data/lib/maltese/version.rb

@@ -1,3 +1,3 @@
 module Maltese
-  VERSION = "0.9.1"
+  VERSION = "0.9.2"
 end

data/maltese.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'activesupport', '>= 4.2.5', '< 6'
   s.add_dependency 'dotenv', '~> 2.1', '>= 2.1.1'
   s.add_dependency 'thor', '~> 0.19'
+  s.add_dependency 'retriable', '~> 3.1'
   s.add_dependency 'sitemap_generator', '~> 6.0'
   s.add_dependency 'aws-sdk-s3', '~> 1.19'
   s.add_dependency 'mime-types', '~> 3.1'

data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_catch_bad_request_errors_with_the_Datacite_REST_API.yml

@@ -0,0 +1,59 @@
+---
+http_interactions:
+- request:
+    method: put
+    uri: https://s3.eu-west-1.amazonaws.com/search.test.datacite.org/sitemaps/sitemap.xml.gz
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIANy18F0AA5WSTU+EMBCG/wrp1dAP3IOQbvfmaT25Jl5rqdCktMjUhf33dgtr0HjAG8w8z8xLGH6YOpud9QDGuz1imKJMO+Vr45o9ejk95g/oIPjnYEGHLLIOqgnMHrUh9BUh4zji8R77oSEFpYy8Ph2fVas7mRsHQTqlURb5ClLx6JUMadFKBxNiq4c0ZOaALEVCcZn9g7094wlqNMf9167FqUwnG/3DbLxvrMbKd7+9PMGEYXazz6bWfqud4LXt9Ahb5Su7zt35N2M3B59pkn767PfxS2Jn64AFX0+Y2tDZv86DlWVJUhelexLceiWuHEQQtBxUi4OGgGsZpIobksbJFeNWQuh8LQrKypwVOWMnWlbFrtrt7iitKI3cgnDVStfo90F/CGlHeQFOViXeD8YPJlxETM3J9xsnKRSZT118AY2t+lEZAwAA
+    headers:
+      Content-Type:
+      - application/x-gzip
+      Accept-Encoding:
+      - ''
+      User-Agent:
+      - aws-sdk-ruby3/3.85.0 ruby/2.6.3 universal.x86_64-darwin19 aws-sdk-s3/1.59.0
+      X-Amz-Acl:
+      - public-read
+      Cache-Control:
+      - private, max-age=0, no-cache
+      Expect:
+      - 100-continue
+      Content-Md5:
+      - lxwQjxbjrf0xGMXvjTOX9g==
+      X-Amz-Date:
+      - 20191211T092444Z
+      X-Amz-Content-Sha256:
+      - 30bde17d18b8abe11ac80301be4511cf1e773a7c993caf77a08f9d216615e7b7
+      Authorization:
+      - AWS4-HMAC-SHA256 Credential=AKIAJAMMCXAR3IXMNCGQ/20191211/eu-west-1/s3/aws4_request,
+        SignedHeaders=cache-control;content-md5;content-type;expect;host;user-agent;x-amz-acl;x-amz-content-sha256;x-amz-date,
+        Signature=99527ec52cae5635e3928767cf9743f88757fc727cf3be085e99071c3959856b
+      Content-Length:
+      - '333'
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Amz-Id-2:
+      - 9SEmB+KZX4UNyualN/z8RPkrjhNZQOrxEREvMwCzXGB35F6gZty/R+QADhxlFJxORz9cYmp9jcs=
+      X-Amz-Request-Id:
+      - 2ED234E38358C809
+      Date:
+      - Wed, 11 Dec 2019 09:24:48 GMT
+      Etag:
+      - '"971c108f16e3adfd3118c5ef8d3397f6"'
+      Content-Length:
+      - '0'
+      Server:
+      - AmazonS3
+    body:
+      encoding: UTF-8
+      string: ''
+    http_version: 
+  recorded_at: Wed, 11 Dec 2019 09:24:48 GMT
+recorded_with: VCR 3.0.3

data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_handle_bad_request_errors_with_the_Datacite_REST_API.yml

@@ -0,0 +1,59 @@
+---
+http_interactions:
+- request:
+    method: put
+    uri: https://s3.eu-west-1.amazonaws.com/search.test.datacite.org/sitemaps/sitemap.xml.gz
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIACe28F0AA5WSTU+EMBCG/wrp1dAPzCZCut2bp/XkmnitpUKT0iJTF/bf2y2sQeMBbzDzPDMvYfhh6mx21gMY7/aIYYoy7ZSvjWv26OX0mD+gg+CfgwUdssg6qCYwe9SG0FeEjOOIx3vsh4YUlDLy+nR8Vq3uZG4cBOmURlnkK0jFo1cypEUrHUyIrR7SkJkDshQJxWX2D/b2jCeo0Rz3X7sWpzKdbPQPs/G+sRor3/328gQThtnNPpta+612gte20yNsla/sOnfn34zdHHymSfrps9/HL4mdrQMWfD1hakNn/zoPVpYlSV2U7klw65W4chBB0HJQLQ4aAq5lkCpuSBonV4xbCaHztSgoK3NW5IydaFkVu2pX3lFaURq5BeGqla7R74P+ENKO8gKcrEq8H4wfTLiImJqT7zdOUigyn7r4AuvlHP0ZAwAA
+    headers:
+      Content-Type:
+      - application/x-gzip
+      Accept-Encoding:
+      - ''
+      User-Agent:
+      - aws-sdk-ruby3/3.85.0 ruby/2.6.3 universal.x86_64-darwin19 aws-sdk-s3/1.59.0
+      X-Amz-Acl:
+      - public-read
+      Cache-Control:
+      - private, max-age=0, no-cache
+      Expect:
+      - 100-continue
+      Content-Md5:
+      - 6kerbfCyNd6pPzaKoM2YfA==
+      X-Amz-Date:
+      - 20191211T092559Z
+      X-Amz-Content-Sha256:
+      - 8e44f92f3f60740ecea4fa2484527a432755024ddb3217a9594f7ad503653554
+      Authorization:
+      - AWS4-HMAC-SHA256 Credential=AKIAJAMMCXAR3IXMNCGQ/20191211/eu-west-1/s3/aws4_request,
+        SignedHeaders=cache-control;content-md5;content-type;expect;host;user-agent;x-amz-acl;x-amz-content-sha256;x-amz-date,
+        Signature=36ccdaef2b69793980e844b45fdd993ed3733c029b0ea41203666025dc6bfc7c
+      Content-Length:
+      - '333'
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Amz-Id-2:
+      - n6qSeGOxeKGqA0EKgBzqo6Uerr9zBo5MWnpviel/xBg+wUWtujTI8mJy0jVhDBB6xrXD19QKMNc=
+      X-Amz-Request-Id:
+      - 4DEC13308960594B
+      Date:
+      - Wed, 11 Dec 2019 09:26:00 GMT
+      Etag:
+      - '"ea47ab6df0b235dea93f368aa0cd987c"'
+      Content-Length:
+      - '0'
+      Server:
+      - AmazonS3
+    body:
+      encoding: UTF-8
+      string: ''
+    http_version: 
+  recorded_at: Wed, 11 Dec 2019 09:25:59 GMT
+recorded_with: VCR 3.0.3

data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_handle_timeout_errors_with_the_Datacite_REST_API.yml

@@ -0,0 +1,59 @@
+---
+http_interactions:
+- request:
+    method: put
+    uri: https://s3.eu-west-1.amazonaws.com/search.test.datacite.org/sitemaps/sitemap.xml.gz
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIACi28F0AA5WSTU+EMBBA/wrp1dAPTIyQbvfmaT25Jl5rqdCktMjUhf33dgtr0HhgL6TMvDczTYfvp85mJz2A8W6HGKYo00752rhmh16PT/kj2gv+NVjQIYusg2oCs0NtCH1FyDiOeLzHfmhIQSkjb8+HF9XqTubGQZBOaZRFvoIUPHglQ2q00sGEmOohFZk5IEuQUFxmN7DXM56gRvO4N/VanMp0stG/zMb7xmqsfPfXyxNMGGZX+2Rq7bfaCV7bTo+wVb6w67k7/27s5sFnmqRHn/0+3iRmthZY8HWFqQ2d/W89WFmWJGVR2ifBrVfiwkEEQctBtThoCLiWQarYIWmcXDBuJYTO16KgrMxZkTN2pGVVPFSU3lEav5FbEK5a6Rr9MehPIe0oz8DJKsT7wfjBhLOIU3Py88dJGorMqy6+AQQYhhIZAwAA
+    headers:
+      Content-Type:
+      - application/x-gzip
+      Accept-Encoding:
+      - ''
+      User-Agent:
+      - aws-sdk-ruby3/3.85.0 ruby/2.6.3 universal.x86_64-darwin19 aws-sdk-s3/1.59.0
+      X-Amz-Acl:
+      - public-read
+      Cache-Control:
+      - private, max-age=0, no-cache
+      Expect:
+      - 100-continue
+      Content-Md5:
+      - K+m6HAwodaexD99LvSm33Q==
+      X-Amz-Date:
+      - 20191211T092600Z
+      X-Amz-Content-Sha256:
+      - fd101b8ee345e029935ddd61b64411b3985312a7e150c2f8c387d55fdabf9b70
+      Authorization:
+      - AWS4-HMAC-SHA256 Credential=AKIAJAMMCXAR3IXMNCGQ/20191211/eu-west-1/s3/aws4_request,
+        SignedHeaders=cache-control;content-md5;content-type;expect;host;user-agent;x-amz-acl;x-amz-content-sha256;x-amz-date,
+        Signature=0d11f0f3b8f9499eeaa40c80d540a173f3737a2ef7e452da15af1241f033f304
+      Content-Length:
+      - '333'
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Amz-Id-2:
+      - pALe39nPehX2z3S/kU9vYTlDBVdZgLtZVs2kzkT+5gs/x1ubsVa3GbONCdTOG9CjJy2OH6SlfHU=
+      X-Amz-Request-Id:
+      - 4FA1F1258A28740E
+      Date:
+      - Wed, 11 Dec 2019 09:26:02 GMT
+      Etag:
+      - '"2be9ba1c0c2875a7b10fdf4bbd29b7dd"'
+      Content-Length:
+      - '0'
+      Server:
+      - AmazonS3
+    body:
+      encoding: UTF-8
+      string: ''
+    http_version: 
+  recorded_at: Wed, 11 Dec 2019 09:26:01 GMT
+recorded_with: VCR 3.0.3

data/spec/sitemap_spec.rb

@@ -33,8 +33,20 @@ describe Maltese::Sitemap, vcr: true do
   end
 
   context "process_data" do
-    it "should catch timeout errors with the Datacite REST API" do
-      stub = stub_request(:get, subject.get_query_url).to_return(:status => [408])
+    it "should handle timeout errors with the Datacite REST API" do
+      stub = stub_request(:get, subject.get_query_url).and_return({ status: [408] }, { status: [408] }, { status: [200] })
+      response = subject.process_data(total: 10, url: subject.get_query_url)
+      expect(response).to eq(1)
+    end
+
+    it "should handle bad request errors with the Datacite REST API" do
+      stub = stub_request(:get, subject.get_query_url).and_return({ status: [502] }, { status: [200] })
+      response = subject.process_data(total: 10, url: subject.get_query_url)
+      expect(response).to eq(1)
+    end
+
+    it "should retry 2 times for bad request errors with the Datacite REST API" do
+      stub = stub_request(:get, subject.get_query_url).and_return({ status: [502] }, { status: [502] }, { status: [502] })
       response = subject.process_data(total: 10, url: subject.get_query_url)
       expect(response).to eq(0)
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: maltese
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.2
 platform: ruby
 authors:
 - Martin Fenner
@@ -106,6 +106,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: retriable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: sitemap_generator
   requirement: !ruby/object:Gem::Requirement
@@ -308,6 +322,9 @@ files:
 - spec/fixtures/vcr_cassettes/Maltese_CLI/sitemap/should_succeed.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_data/should_report_if_there_are_works_returned_by_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_total/with_works.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_catch_bad_request_errors_with_the_Datacite_REST_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_handle_bad_request_errors_with_the_Datacite_REST_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_handle_timeout_errors_with_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_no_works_returned_by_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_works_returned_by_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/queue_jobs/should_report_if_there_are_works_returned_by_the_Datacite_REST_API.yml