maltese 0.9.2 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b25d7aed1fa4b41a642a8bf8449e067c67fa28861cf319c54d24b4ffccbcd6d
-  data.tar.gz: af043d259c92a902d5a42c281a0746fc2cf65f356ebb2bec161cc707f89f9b3a
+  metadata.gz: fe93930c90b5ddc8d0b3b89de7e18b7644356b64f82c76451b16cf9395e94269
+  data.tar.gz: 208578435824ca3f9a9cf2582e1dd73f6df8e018319494469c2a1f2408d9a3ed
 SHA512:
-  metadata.gz: e1399ede091a478151153b5071b7881ae1bedd9068ac017ab34c1cfcf1931559a607f0f2ccba83b1018b3a251898e4498d8557267905478c7dd48e322b288722
-  data.tar.gz: 50cd6f3a1ee8a4b0c508805720efcc58db31208dd7842668b6785cf0684bd03059d2872113e002d4c89c5e501fb2924f6323ff24caa3db5227c86517fbbef7c3
+  metadata.gz: 6dbeb4a4e96a050c987746be6a414dfe53428ed7b30ad211dee2a11df7052ce995f1f07841ce0cc32c1cd9d8904b4197c049b9ba3da7614659fc75adfa55f313
+  data.tar.gz: 8dea3a1fb90e4d9cd94cfa85bb2b4e1263b0635ad3010c937d311ab4519da49c8759a8679edf1313305f67843428be780b9db991ae06ac97e8c4878d4ae5f74b

data/Dockerfile

@@ -11,6 +11,6 @@ RUN apt-get update && apt-get upgrade -y -o Dpkg::Options::="--force-confold" &&
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Install maltese gem
-RUN /sbin/setuser app gem install maltese -v 0.9.2
+RUN /sbin/setuser app gem install maltese -v 0.9.3
 
-CMD maltese sitemap --sitemap_bucket $SITEMAP_BUCKET --rack_env $RACK_ENV --access_key $AWS_ACCESS_KEY_ID --secret_key $AWS_SECRET_ACCESS_KEY --region $AWS_REGION
+CMD maltese sitemap --sitemap_bucket $SITEMAP_BUCKET --rack_env $RACK_ENV --access_key $AWS_ACCESS_KEY_ID --secret_key $AWS_SECRET_ACCESS_KEY --region $AWS_REGION --slack_webhook_url $SLACK_WEBHOOK_URL

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    maltese (0.9.2)
+    maltese (0.9.3)
       activesupport (>= 4.2.5, < 6)
       aws-sdk-s3 (~> 1.19)
       dotenv (~> 2.1, >= 2.1.1)
@@ -11,6 +11,7 @@ PATH
       mime-types (~> 3.1)
       retriable (~> 3.1)
       sitemap_generator (~> 6.0)
+      slack-notifier (~> 2.1)
       thor (~> 0.19)
 
 GEM
@@ -59,7 +60,7 @@ GEM
     i18n (1.7.0)
       concurrent-ruby (~> 1.0)
     jmespath (1.4.0)
-    json (2.2.0)
+    json (2.3.0)
     logstash-event (1.2.02)
     logstash-logger (0.26.1)
       logstash-event (~> 1.2)
@@ -111,6 +112,7 @@ GEM
     simplecov-html (0.10.2)
     sitemap_generator (6.0.2)
       builder (~> 3.0)
+    slack-notifier (2.3.2)
     thor (0.20.3)
     thread_safe (0.3.6)
     tzinfo (1.2.5)

data/lib/maltese/cli.rb

@@ -23,6 +23,7 @@ module Maltese
     method_option :access_key, type: :string
     method_option :secret_key, type: :string
     method_option :region, type: :string
+    method_option :slack_webhook_url, type: :string
     def sitemap
       sitemap = Maltese::Sitemap.new(options)
       sitemap.queue_jobs

data/lib/maltese/sitemap.rb

@@ -1,11 +1,12 @@
 require 'logstash-logger'
 require 'retriable'
+require 'slack-notifier'
 
 module Maltese
   class ::BadGatewayError < StandardError; end
 
   class Sitemap
-    attr_reader :sitemap_bucket, :rack_env, :access_key, :secret_key, :region, :logger
+    attr_reader :sitemap_bucket, :rack_env, :access_key, :secret_key, :region, :slack_webhook_url, :logger
 
     # load ENV variables from .env file if it exists
     env_file = File.expand_path("../../../.env", __FILE__)
@@ -22,12 +23,16 @@ module Maltese
       env_vars.each { |k, v| ENV[k] = v }
     end
 
+    # icon for Slack messages
+    SLACK_ICON_URL = "https://github.com/datacite/segugio/blob/master/source/images/fabrica.png"
+
     def initialize(attributes={})
       @sitemap_bucket = attributes[:sitemap_bucket].presence || "search.test.datacite.org"
       @rack_env = attributes[:rack_env].presence || ENV['RACK_ENV'] || "stage"
       @access_key = attributes[:access_key].presence || ENV['AWS_ACCESS_KEY_ID']
       @secret_key = attributes[:secret_key].presence || ENV['AWS_SECRET_ACCESS_KEY']
       @region = attributes[:region].presence || ENV['AWS_REGION']
+      @slack_webhook_url = attributes[:slack_webhook_url].presence || ENV['SLACK_WEBHOOK_URL']
 
       @logger = LogStashLogger.new(type: :stdout)
     end
@@ -36,6 +41,10 @@ module Maltese
       rack_env == "production" ? "https://search.datacite.org/" : "https://search.test.datacite.org/"
     end
 
+    def slack_title
+      rack_env == "production" ? "DataCite Fabrica" : "DataCite Fabrica Test"
+    end
+
     def sitemaps_path
       "sitemaps/"
     end
@@ -130,8 +139,13 @@ module Maltese
             options[:url] = nil
           end
         rescue => exception
-          logger.error "Error: #{exception.message}."
+          logger.error "Error: #{exception.message}"
           error_count += 1
+          fields = [
+            { title: "Error", value: exception.message },
+            { title: "Time Taken", value: "#{((Time.now - options[:start_time])/ 60.0).ceil} min", short: true }
+          ]
+          send_notification_to_slack(nil, title: slack_title + ": Sitemaps Not Updated", level: "danger", fields: fields) unless rack_env == "test"
           options[:url] = nil
         ensure
           # don't loop when testing
@@ -151,7 +165,7 @@ module Maltese
     def parse_data(result)
       Array.wrap(result.body.fetch("data", nil)).each do |item|
         loc = "/works/" + item.dig("attributes", "doi")
-        sitemap.add loc, changefreq: "monthly", lastmod: item.dig("attributes", "updated")
+        sitemap.add loc, changefreq: "weekly", lastmod: item.dig("attributes", "updated")
       end
       sitemap.sitemap.link_count
     end
@@ -160,7 +174,36 @@ module Maltese
       sitemap.finalize!
       options[:start_time] ||= Time.now
       sitemap.sitemap_index.stats_summary(:time_taken => Time.now - options[:start_time])
+      
+      fields = [
+        { title: "URL", value: "#{sitemap_url}sitemaps/sitemap.xml.gz" },
+        { title: "Number of DOIs", value: format_number(sitemap.sitemap.link_count), short: true },
+        { title: "Time Taken", value: "#{((Time.now - options[:start_time])/ 60.0).ceil} min", short: true }
+      ]
+      send_notification_to_slack(nil, title: slack_title + ": Sitemaps Updated", level: "good", fields: fields) unless rack_env == "test"
       sitemap.sitemap.link_count
     end
+
+    def send_notification_to_slack(text, options={})
+      return nil unless slack_webhook_url.present?
+
+      attachment = {
+        title: options[:title] || "Fabrica Message",
+        text: text,
+        color: options[:level] || "good",
+        fields: options[:fields]
+      }.compact
+
+      notifier = Slack::Notifier.new slack_webhook_url,
+                                     username: "Fabrica",
+                                     icon_url: SLACK_ICON_URL
+      response = notifier.ping attachments: [attachment]
+      response.first.body
+    end
+
+    # from https://codereview.stackexchange.com/questions/28054/separate-numbers-with-commas
+    def format_number(number)
+      number.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+    end
   end
 end

data/lib/maltese/version.rb

@@ -1,3 +1,3 @@
 module Maltese
-  VERSION = "0.9.2"
+  VERSION = "0.9.3"
 end

data/maltese.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'logstash-logger', '~> 0.26.1'
   s.add_dependency 'activesupport', '>= 4.2.5', '< 6'
   s.add_dependency 'dotenv', '~> 2.1', '>= 2.1.1'
+  s.add_dependency 'slack-notifier', '~> 2.1'
   s.add_dependency 'thor', '~> 0.19'
   s.add_dependency 'retriable', '~> 3.1'
   s.add_dependency 'sitemap_generator', '~> 6.0'

data/spec/cli_spec.rb

@@ -16,7 +16,7 @@ describe Maltese::CLI do
       sitemap = Zlib::GzipReader.open("public/sitemaps/sitemap.xml.gz") { |gz| gz.read }
       doc = Nokogiri::XML(sitemap)
       expect(doc.xpath("//xmlns:url").size).to eq(1001)
-      expect(doc.xpath("//xmlns:loc").last.text).to eq("https://search.test.datacite.org/works/10.1007/s10620-013-2631-z")
+      expect(doc.xpath("//xmlns:loc").last.text).to eq("https://search.test.datacite.org/works/10.1080/19393210.2019.1576774")
     end
   end
 end