maltese 0.8.13 → 0.8.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63308466e1da9ddab020ebcf67fb2ffe6147b60c9792db14aded530ef349d5bb
-  data.tar.gz: 152b94caa0acb2fbc9a1defa57329740bb873f3129983e5c0fb9cbf992606eef
+  metadata.gz: 3975693a1ddab582888bf8e4628be0e0b1127a4de475afc0f4126fda14cef80e
+  data.tar.gz: f0f57a6a433a36414a853e6e42735577b7e9a40dcd34b9931cd6efb9cd1d664c
 SHA512:
-  metadata.gz: 7ddcd5fcf4b026d3a2fdacca318f330ef76783fc0b0332cb7c4bb927b94aab72b0d49c4cd30fa8173658422fb42ba60b3ccaf2e2882869989926bbc0e673b8d1
-  data.tar.gz: 60d85866dc019bd5c700f64c4242a8ceb5c61ad347aaea8e5d6e877ca629f3dbc4ddff54b1d3a612499e06abae9e9f5adfc398a0611f24891285b02d7bb3534c
+  metadata.gz: a401e5b79ca1f326adb521701a1d355f87f56ef1d7c42122ee0093d3cf2af39597d8b7720eb0b78270f441d7686c0bcf3af250959a212a91340d09f8a032db6d
+  data.tar.gz: 256c0579d4deeb794bae24f6560a216c2953f6ce19f4ab8c94b97a975c19f2949ef696ce8379997fafa068b9888176b0ddeeb1b673b8d5be015c982196422080

data/Dockerfile

@@ -11,6 +11,6 @@ RUN apt-get update && apt-get upgrade -y -o Dpkg::Options::="--force-confold" &&
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Install maltese gem
-RUN /sbin/setuser app gem install maltese -v 0.8.13
+RUN /sbin/setuser app gem install maltese -v 0.8.14
 
 CMD maltese sitemap --sitemap_bucket $SITEMAP_BUCKET

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    maltese (0.8.13)
+    maltese (0.8.14)
       activesupport (>= 4.2.5, < 6)
       aws-sdk-s3 (~> 1.19)
       dotenv (~> 2.1, >= 2.1.1)
@@ -21,13 +21,13 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.0.3)
-    aws-partitions (1.251.0)
-    aws-sdk-core (3.84.0)
+    aws-partitions (1.252.0)
+    aws-sdk-core (3.85.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.26.0)
+    aws-sdk-kms (1.27.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
     aws-sdk-s3 (1.59.0)

data/lib/maltese/sitemap.rb

@@ -90,17 +90,30 @@ module Maltese
 
     def process_data(options = {})
       options[:start_time] = Time.now
+      link_count = 0
+      error_count = 0
 
       # walk through paginated results
       while options[:url] do
         response = get_data(options[:url])
-        parse_data(response)
-        options[:url] = response.body.dig("links", "next")
+
+        if response.status == 200
+          link_count += parse_data(response)
+          puts "#{link_count} DOIs parsed."
+          options[:url] = response.body.dig("links", "next")
+        else
+          puts "An error occured for URL #{options[:url]}:."
+          puts "Error message: #{response.body.fetch("errors").inspect}" if response.body.fetch("errors", nil).present?
+          error_count += 1
+          options[:url] = nil
+        end 
 
         # don't loop when testing
         break if ENV['RACK'] == "test"     
       end
 
+      return link_count if error_count > 0
+
       push_data(options)
     end
 
@@ -109,16 +122,10 @@ module Maltese
     end
 
     def parse_data(result)
-      if result.body.fetch("errors", nil).present?
-        puts "An error occured: #{result.body.fetch("errors").inspect}"
-        return result.body.fetch("errors") 
-      end
-
       result.body.fetch("data", []).each do |item|
         loc = "/works/" + item.dig("attributes", "doi")
         sitemap.add loc, changefreq: "monthly", lastmod: item.dig("attributes", "updated")
       end
-      puts "#{result.body.fetch("data", []).size} DOIs parsed."
       sitemap.sitemap.link_count
     end
 

data/lib/maltese/version.rb

@@ -1,3 +1,3 @@
 module Maltese
-  VERSION = "0.8.13"
+  VERSION = "0.8.14"
 end

data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_catch_timeout_errors_with_the_Datacite_REST_API.yml

@@ -0,0 +1,59 @@
+---
+http_interactions:
+- request:
+    method: put
+    uri: https://s3.eu-west-1.amazonaws.com/search.test.datacite.org/sitemaps/sitemap.xml.gz
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIAD9W710AA5WSPW+DMBCG/wryWuEP0iEgQ7ZO6dRU6uoaFywZm3JuIP++jiEVrTqQDe6e5+5FHD9MnUnOagDtbIkYpihRVrpa26ZEr6endI8OFf8aDCifBNZCMYEuUet9XxAyjiMed9gNDckoZeTt+fgiW9WJVFvwwkqFksAXEItHJ4WPi1Y6aB9aPcQhMwdkKRKK8+QO9vaMJ6jRHPeuXYtT6E406pfZONcYhaXr/npphAnD7Gafda3cVjvCa9uqEbbKV3adu3Pv2mwOPtMk/vTZ78OXhM7WAQu+njC1vjP/nQfL85zELor3VHHjZHXlIICgxCBb7BV4XAsvZNgQNU6uGDcCfOfqKqMsT1mWMnqi+yJ7LHbsgdKC0sAtCJetsI36GNRnJcwoLsDJqsT7QbtB+0sVUnPy88ZJDEXmU6++AT8Dg9QZAwAA
+    headers:
+      Content-Type:
+      - application/x-gzip
+      Accept-Encoding:
+      - ''
+      User-Agent:
+      - aws-sdk-ruby3/3.85.0 ruby/2.6.3 universal.x86_64-darwin19 aws-sdk-s3/1.59.0
+      X-Amz-Acl:
+      - public-read
+      Cache-Control:
+      - private, max-age=0, no-cache
+      Expect:
+      - 100-continue
+      Content-Md5:
+      - qw5jsYX5Zml0Xk8/J5o/pw==
+      X-Amz-Date:
+      - 20191210T082431Z
+      X-Amz-Content-Sha256:
+      - c1b9ed2122fe6722ea084cc215ee6b1f866e0f1b1f9cbb70b8d9673bf0b17d43
+      Authorization:
+      - AWS4-HMAC-SHA256 Credential=AKIAJAMMCXAR3IXMNCGQ/20191210/eu-west-1/s3/aws4_request,
+        SignedHeaders=cache-control;content-md5;content-type;expect;host;user-agent;x-amz-acl;x-amz-content-sha256;x-amz-date,
+        Signature=7d5f33f12175ee8b8af02cd03e5d61c36ce3898d1a86660ad1d56ced965508ea
+      Content-Length:
+      - '333'
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Amz-Id-2:
+      - 8ToOQoDXCK0nj5F+9qE7D6Pn4Clf3tfjbW8CpFD/GJlZd13pEktxxKVozf/EQXvigc703t04J0w=
+      X-Amz-Request-Id:
+      - 7041D9503577FFF6
+      Date:
+      - Tue, 10 Dec 2019 08:24:33 GMT
+      Etag:
+      - '"ab0e63b185f96669745e4f3f279a3fa7"'
+      Content-Length:
+      - '0'
+      Server:
+      - AmazonS3
+    body:
+      encoding: UTF-8
+      string: ''
+    http_version: 
+  recorded_at: Tue, 10 Dec 2019 08:24:32 GMT
+recorded_with: VCR 3.0.3

data/spec/sitemap_spec.rb

@@ -32,6 +32,14 @@ describe Maltese::Sitemap, vcr: true do
     end
   end
 
+  context "process_data" do
+    it "should catch timeout errors with the Datacite REST API" do
+      stub = stub_request(:get, subject.get_query_url).to_return(:status => [408])
+      response = subject.process_data(total: 10, url: subject.get_query_url)
+      expect(response).to eq(0)
+    end
+  end
+
   context "get_data" do
     it "should report if there are works returned by the Datacite REST API" do
       response = subject.get_data(subject.get_query_url)
@@ -62,12 +70,6 @@ describe Maltese::Sitemap, vcr: true do
       response = subject.parse_data(result)
       expect(response).to eq(1001)
     end
-
-    it "should catch timeout errors with the Datacite REST API" do
-      result = OpenStruct.new(body: { "errors" => [{ "title" => "the server responded with status 408 for https://REST.test.datacite.org", "status" => 408 }] })
-      response = subject.parse_data(result)
-      expect(response).to eq(result.body["errors"])
-    end
   end
 
   context "push_data" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maltese
 version: !ruby/object:Gem::Version
-  version: 0.8.13
+  version: 0.8.14
 platform: ruby
 authors:
 - Martin Fenner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-09 00:00:00.000000000 Z
+date: 2019-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: maremma
@@ -281,6 +281,7 @@ files:
 - spec/fixtures/vcr_cassettes/Maltese_CLI/sitemap/should_succeed.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_data/should_report_if_there_are_works_returned_by_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_total/with_works.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_catch_timeout_errors_with_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_no_works_returned_by_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_works_returned_by_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/queue_jobs/should_report_if_there_are_works_returned_by_the_Datacite_REST_API.yml