maltese 0.8.14 → 0.8.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3975693a1ddab582888bf8e4628be0e0b1127a4de475afc0f4126fda14cef80e
-  data.tar.gz: f0f57a6a433a36414a853e6e42735577b7e9a40dcd34b9931cd6efb9cd1d664c
+  metadata.gz: 47f0ba0813eeed3c56a5636d34acbb9bc3349cef65513b3a834c05dcda19bc08
+  data.tar.gz: 6ecb5ff42d4759e07c7632420aa78a81284a57ebba7aeca9d2747218e0d78297
 SHA512:
-  metadata.gz: a401e5b79ca1f326adb521701a1d355f87f56ef1d7c42122ee0093d3cf2af39597d8b7720eb0b78270f441d7686c0bcf3af250959a212a91340d09f8a032db6d
-  data.tar.gz: 256c0579d4deeb794bae24f6560a216c2953f6ce19f4ab8c94b97a975c19f2949ef696ce8379997fafa068b9888176b0ddeeb1b673b8d5be015c982196422080
+  metadata.gz: 05aedc2d77881876ac08bf9f11a2796220619e06e128209f83600758d5c5c5448a47db3b7a25f67870b1ceaba63ea5c840958c2f5fa1ad85666280b285362306
+  data.tar.gz: c15bcefd230f7839556345f455879c702ed25a06368410d9e6e1bc72a86885b4cb06fdb8ddb03f5c3a21be605339834da56c21ef2c91fc39f8588fa45e84e483

data/Dockerfile

@@ -11,6 +11,6 @@ RUN apt-get update && apt-get upgrade -y -o Dpkg::Options::="--force-confold" &&
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Install maltese gem
-RUN /sbin/setuser app gem install maltese -v 0.8.14
+RUN /sbin/setuser app gem install maltese -v 0.8.15
 
-CMD maltese sitemap --sitemap_bucket $SITEMAP_BUCKET
+CMD maltese sitemap --sitemap_bucket $SITEMAP_BUCKET --rack_env $RACK_ENV --access_key $AWS_ACCESS_KEY_ID --secret_key $AWS_SECRET_ACCESS_KEY --region $AWS_REGION

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    maltese (0.8.14)
+    maltese (0.8.15)
       activesupport (>= 4.2.5, < 6)
       aws-sdk-s3 (~> 1.19)
       dotenv (~> 2.1, >= 2.1.1)
+      faraday (= 0.17.0)
       maremma (~> 4.1)
       mime-types (~> 3.1)
       sitemap_generator (~> 6.0)
@@ -46,7 +47,7 @@ GEM
     docile (1.1.5)
     dotenv (2.7.5)
     excon (0.67.0)
-    faraday (0.17.1)
+    faraday (0.17.0)
       multipart-post (>= 1.2, < 3)
     faraday-encoding (0.0.5)
       faraday

data/README.md

@@ -4,7 +4,7 @@
 
 # Maltese
 
-Ruby gem and command-line tool for generating sitemap files from the DataCite REST API. Uses the [SitemapGenerator](https://github.com/kjvarga/sitemap_generator) gem and can be run as Docker container, e.g. using ECS and triggered by AWS Lambda, as described [here](https://medium.com/@pahud/ecs-task-runner-with-lambda-4594b72ccb#.5xpmf2inz).
+Ruby gem and command-line tool for generating sitemap files from the DataCite REST API. Uses the [SitemapGenerator](https://github.com/kjvarga/sitemap_generator) gem and can be run as Docker container, e.g. using as a scheduled task in AWS ECS triggered by AWS Cloudwatch Events.
 
 Run as a command-line tool:
 

data/lib/maltese/cli.rb

@@ -19,6 +19,10 @@ module Maltese
 
     desc "sitemap", "generate sitemap for DataCite Search"
     method_option :sitemap_bucket, type: :string
+    method_option :rack_env, type: :string
+    method_option :access_key, type: :string
+    method_option :secret_key, type: :string
+    method_option :region, type: :string
     def sitemap
       sitemap = Maltese::Sitemap.new(options)
       sitemap.queue_jobs

data/lib/maltese/sitemap.rb

@@ -1,6 +1,6 @@
 module Maltese
   class Sitemap
-    attr_reader :sitemap_bucket
+    attr_reader :sitemap_bucket, :rack_env, :access_key, :secret_key, :region
 
     # load ENV variables from .env file if it exists
     env_file = File.expand_path("../../../.env", __FILE__)
@@ -19,10 +19,14 @@ module Maltese
 
     def initialize(attributes={})
       @sitemap_bucket = attributes[:sitemap_bucket].presence || "search.test.datacite.org"
+      @rack_env = attributes[:rack_env].presence || ENV['RACK_ENV'] || "stage"
+      @access_key = attributes[:access_key].presence || ENV['AWS_ACCESS_KEY_ID']
+      @secret_key = attributes[:secret_key].presence || ENV['AWS_SECRET_ACCESS_KEY']
+      @region = attributes[:region].presence || ENV['AWS_REGION']
     end
 
     def sitemap_url
-      ENV['RACK_ENV'] == "production" ? "https://search.datacite.org/" : "https://search.test.datacite.org/"
+      rack_env == "production" ? "https://search.datacite.org/" : "https://search.test.datacite.org/"
     end
 
     def sitemaps_path
@@ -30,7 +34,7 @@ module Maltese
     end
 
     def search_path
-      ENV['RACK_ENV'] == "production" ? "https://api.datacite.org/dois?" : "https://api.test.datacite.org/dois?"
+      rack_env == "production" ? "https://api.datacite.org/dois?" : "https://api.test.datacite.org/dois?"
     end
 
     def timeout
@@ -52,22 +56,24 @@ module Maltese
 
     def s3_adapter
       SitemapGenerator::AwsSdkAdapter.new(sitemap_bucket,
-                                      aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
-                                      aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
-                                      aws_region: ENV['AWS_REGION'])
+                                      aws_access_key_id: access_key,
+                                      aws_secret_access_key: secret_key,
+                                      aws_region: region)
     end
 
     def queue_jobs(options={})
       total = get_total(options)
 
-      if total > 0
-        puts process_data(options.merge(total: total, url: get_query_url))
+      if total.nil?
+        puts "An error occured."
+      elsif total > 0
+        process_data(options.merge(total: total, url: get_query_url))
       else
         puts "No works found."
       end
 
       # return number of works queued
-      total
+      total.to_i
     end
 
     def get_total(options={})
@@ -102,14 +108,14 @@ module Maltese
           puts "#{link_count} DOIs parsed."
           options[:url] = response.body.dig("links", "next")
         else
-          puts "An error occured for URL #{options[:url]}:."
+          puts "An error occured for URL #{options[:url]}."
           puts "Error message: #{response.body.fetch("errors").inspect}" if response.body.fetch("errors", nil).present?
           error_count += 1
           options[:url] = nil
         end 
 
         # don't loop when testing
-        break if ENV['RACK'] == "test"     
+        break if rack_env == "test"     
       end
 
       return link_count if error_count > 0
@@ -122,7 +128,7 @@ module Maltese
     end
 
     def parse_data(result)
-      result.body.fetch("data", []).each do |item|
+      Array.wrap(result.body.fetch("data", nil)).each do |item|
         loc = "/works/" + item.dig("attributes", "doi")
         sitemap.add loc, changefreq: "monthly", lastmod: item.dig("attributes", "updated")
       end

data/lib/maltese/version.rb

@@ -1,3 +1,3 @@
 module Maltese
-  VERSION = "0.8.14"
+  VERSION = "0.8.15"
 end

data/maltese.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |s|
 
   # Declary dependencies here, rather than in the Gemfile
   s.add_dependency 'maremma', '~> 4.1'
+  s.add_dependency 'faraday', '0.17.0'
   s.add_dependency 'activesupport', '>= 4.2.5', '< 6'
   s.add_dependency 'dotenv', '~> 2.1', '>= 2.1.1'
   s.add_dependency 'thor', '~> 0.19'

data/spec/cli_spec.rb

@@ -7,7 +7,7 @@ describe Maltese::CLI do
   end
 
   let(:sitemap_bucket) { "search.test.datacite.org" }
-  let(:cli_options) { { sitemap_bucket: sitemap_bucket } }
+  let(:cli_options) { { sitemap_bucket: sitemap_bucket, rack_env: "test" } }
 
   describe "sitemap", vcr: true, :order => :defined do
     it 'should succeed' do
@@ -16,7 +16,7 @@ describe Maltese::CLI do
       sitemap = Zlib::GzipReader.open("public/sitemaps/sitemap.xml.gz") { |gz| gz.read }
       doc = Nokogiri::XML(sitemap)
       expect(doc.xpath("//xmlns:url").size).to eq(1001)
-      expect(doc.xpath("//xmlns:loc").last.text).to eq("https://search.test.datacite.org/works/10.17557/tjfc.424379")
+      expect(doc.xpath("//xmlns:loc").last.text).to eq("https://search.test.datacite.org/works/10.1007/s10620-013-2631-z")
     end
   end
 end