maltese 0.2.4 → 0.8

data/.travis.yml

@@ -1,6 +1,6 @@
 language: ruby
 rvm:
-- 2.3.3
+- 2.4.4
 
 addons:
   code_climate:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    maltese (0.2.4)
+    maltese (0.8)
       activesupport (>= 4.2.5, < 6)
       aws-sdk-s3 (~> 1.19)
       dotenv (~> 2.1, >= 2.1.1)
@@ -13,50 +13,51 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.1)
+    activesupport (5.2.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    addressable (2.5.2)
+    addressable (2.6.0)
       public_suffix (>= 2.0.2, < 4.0)
-    aws-eventstream (1.0.1)
-    aws-partitions (1.103.0)
-    aws-sdk-core (3.27.0)
-      aws-eventstream (~> 1.0)
+    aws-eventstream (1.0.3)
+    aws-partitions (1.170.0)
+    aws-sdk-core (3.54.1)
+      aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1.0)
-      aws-sigv4 (~> 1.0)
+      aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.9.0)
-      aws-sdk-core (~> 3, >= 3.26.0)
-      aws-sigv4 (~> 1.0)
-    aws-sdk-s3 (1.19.0)
-      aws-sdk-core (~> 3, >= 3.26.0)
+    aws-sdk-kms (1.21.0)
+      aws-sdk-core (~> 3, >= 3.53.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-s3 (1.41.0)
+      aws-sdk-core (~> 3, >= 3.53.0)
       aws-sdk-kms (~> 1)
-      aws-sigv4 (~> 1.0)
-    aws-sigv4 (1.0.3)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.1.0)
+      aws-eventstream (~> 1.0, >= 1.0.2)
     builder (3.2.3)
-    codeclimate-test-reporter (1.0.8)
+    codeclimate-test-reporter (1.0.9)
       simplecov (<= 0.13)
-    concurrent-ruby (1.0.5)
+    concurrent-ruby (1.1.5)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     diff-lcs (1.3)
     docile (1.1.5)
-    dotenv (2.5.0)
-    excon (0.62.0)
-    faraday (0.15.2)
+    dotenv (2.7.2)
+    excon (0.64.0)
+    faraday (0.15.4)
       multipart-post (>= 1.2, < 3)
-    faraday-encoding (0.0.4)
+    faraday-encoding (0.0.5)
       faraday
     faraday_middleware (0.12.2)
       faraday (>= 0.7.4, < 1.0)
-    hashdiff (0.3.7)
-    i18n (1.1.0)
+    hashdiff (0.4.0)
+    i18n (1.6.0)
       concurrent-ruby (~> 1.0)
     jmespath (1.4.0)
-    json (2.1.0)
-    maremma (4.1.1)
+    json (2.2.0)
+    maremma (4.2.1)
       activesupport (>= 4.2.5, < 6)
       addressable (>= 2.3.6)
       builder (~> 3.2, >= 3.2.2)
@@ -69,46 +70,46 @@ GEM
       oj (>= 2.8.3)
     mime-types (3.2.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2018.0812)
+    mime-types-data (3.2019.0331)
     mini_portile2 (2.3.0)
     minitest (5.11.3)
     multi_json (1.13.1)
-    multipart-post (2.0.0)
-    nokogiri (1.8.4)
+    multipart-post (2.1.1)
+    nokogiri (1.8.5)
       mini_portile2 (~> 2.3.0)
-    oj (3.6.8)
-    public_suffix (3.0.3)
-    rack (2.0.5)
+    oj (3.7.12)
+    public_suffix (3.1.0)
+    rack (2.0.7)
     rack-test (0.8.3)
       rack (>= 1.0, < 3)
-    rake (12.3.1)
+    rake (12.3.2)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
     rspec-core (3.8.0)
       rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.1)
+    rspec-expectations (3.8.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-mocks (3.8.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    safe_yaml (1.0.4)
+    safe_yaml (1.0.5)
     simplecov (0.13.0)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    sitemap_generator (6.0.1)
+    sitemap_generator (6.0.2)
       builder (~> 3.0)
-    thor (0.20.0)
+    thor (0.20.3)
     thread_safe (0.3.6)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
     vcr (3.0.3)
-    webmock (3.4.2)
+    webmock (3.5.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
@@ -128,4 +129,4 @@ DEPENDENCIES
   webmock (~> 3.0, >= 3.0.1)
 
 BUNDLED WITH
-   1.16.1
+   1.17.3

data/README.md

@@ -9,7 +9,7 @@ Ruby gem and command-line tool for generating sitemap files from the DataCite RE
 Run as a command-line tool:
 
 ```
-maltese sitemap --from_date 2017-02-15
+maltese sitemap
 ```
 
 ## Installation

data/lib/maltese/sitemap.rb

@@ -21,8 +21,6 @@ module Maltese
       @sitemap_bucket = attributes[:sitemap_bucket].presence || "search.test.datacite.org"
       @from_date = attributes[:from_date].presence || (Time.now.to_date - 1.day).iso8601
       @until_date = attributes[:until_date].presence || Time.now.to_date.iso8601
-      @solr_username = ENV['SOLR_USERNAME']
-      @solr_password = ENV['SOLR_PASSWORD']
     end
 
     def sitemap_url
@@ -34,7 +32,7 @@ module Maltese
     end
 
     def search_path
-      ENV['RACK_ENV'] == "production" ? "https://solr.datacite.org/api?" : "https://solr.test.datacite.org/api?"
+      ENV['RACK_ENV'] == "production" ? "https://api.datacite.org/dois?" : "https://api.test.datacite.org/dois?"
     end
 
     def timeout
@@ -42,7 +40,7 @@ module Maltese
     end
 
     def job_batch_size
-      50000
+      1000
     end
 
     def sitemap
@@ -67,7 +65,7 @@ module Maltese
       if total > 0
         puts process_data(options.merge(total: total))
       else
-        puts "No works found for date range #{from_date} - #{until_date}."
+        puts "No works found."
       end
 
       # return number of works queued
@@ -75,28 +73,20 @@ module Maltese
     end
 
     def get_total(options={})
-      query_url = get_query_url(options.merge(rows: 0))
-      # Add basic auth options in
-      options = options.merge(username: @solr_username, password: @solr_password)
+      query_url = get_query_url(options.merge(size: 0))
 
       result = Maremma.get(query_url, options)
-      result.body.fetch("data", {}).fetch("response", {}).fetch("numFound", 0)
+      result.body.dig("meta", "total")
     end
 
     def get_query_url(options={})
-      options[:offset] = options[:offset].to_i || 0
-      options[:rows] = options[:rows].presence || job_batch_size
-
-      updated = "updated:[#{from_date}T00:00:00Z TO #{until_date}T23:59:59Z]"
-      fq = "#{updated} AND has_metadata:true AND is_active:true"
-
-      params = { q: "*:*",
-                 fq: fq,
-                 start: options[:offset],
-                 rows: options[:rows],
-                 fl: "doi,updated",
-                 sort: "updated asc",
-                 wt: "json"}
+      options[:cursor] = options[:cursor] || 1
+      options[:size] = options[:size] || job_batch_size
+
+      params = { 
+        "page[cursor]": options[:cursor],
+        "page[size]": options[:size],
+      }
       search_path + URI.encode_www_form(params)
     end
 
@@ -104,11 +94,9 @@ module Maltese
       options[:start_time] = Time.now
 
       # walk through paginated results
-      total_pages = (options[:total].to_f / job_batch_size).ceil
-
-      (0...total_pages).each do |page|
-        options[:offset] = page * job_batch_size
+      while options[:cursor] do
         data = get_data(options.merge(timeout: timeout))
+        options[:cursor] = data.dig("links", "next")
         parse_data(data)
       end
 
@@ -118,19 +106,15 @@ module Maltese
     def get_data(options={})
       query_url = get_query_url(options)
 
-      # Add basic auth options in
-      options = options.merge(username: @solr_username, password: @solr_password)
-
       Maremma.get(query_url, options)
     end
 
     def parse_data(result)
       return result.body.fetch("errors") if result.body.fetch("errors", nil).present?
 
-      items = result.body.fetch("data", {}).fetch('response', {}).fetch('docs', nil)
-      Array(items).each do |item|
-        loc = "/works/" + item.fetch("doi")
-        sitemap.add loc, changefreq: "monthly", lastmod: item.fetch("updated")
+      result.body.fetch("data", []).each do |item|
+        loc = "/works/" + item.dig("attributes", "doi")
+        sitemap.add loc, changefreq: "monthly", lastmod: item.dig("attrributes", "updated")
       end
       sitemap.sitemap.link_count
     end

data/lib/maltese/version.rb

@@ -1,3 +1,3 @@
 module Maltese
-  VERSION = "0.2.4"
+  VERSION = "0.8"
 end

data/spec/cli_spec.rb

@@ -6,30 +6,17 @@ describe Maltese::CLI do
     described_class.new
   end
 
-  let(:from_date) { "2018-03-15" }
-  let(:until_date) { "2018-04-08" }
   let(:sitemap_bucket) { "search.test.datacite.org" }
-  let(:cli_options) { { sitemap_bucket: sitemap_bucket,
-                        from_date: from_date,
-                        until_date: until_date } }
+  let(:cli_options) { { sitemap_bucket: sitemap_bucket } }
 
   describe "sitemap", vcr: true, :order => :defined do
     it 'should succeed' do
       subject.options = cli_options
-      expect { subject.sitemap }.to output(/266 links/).to_stdout
+      expect { subject.sitemap }.to output(/1 links/).to_stdout
       sitemap = Zlib::GzipReader.open("public/sitemaps/sitemap.xml.gz") { |gz| gz.read }
       doc = Nokogiri::XML(sitemap)
-      expect(doc.xpath("//xmlns:url").size).to eq(266)
-      expect(doc.xpath("//xmlns:loc").last.text).to eq("https://search.test.datacite.org/works/10.0133/37522")
-    end
-
-    it 'should succeed with no works' do
-      from_date = "2005-04-07"
-      until_date = "2005-04-08"
-      subject.options = { sitemap_bucket: sitemap_bucket,
-                          from_date: from_date,
-                          until_date: until_date }
-      expect { subject.sitemap }.to output("No works found for date range 2005-04-07 - 2005-04-08.\n").to_stdout
+      expect(doc.xpath("//xmlns:url").size).to eq(1)
+      expect(doc.xpath("//xmlns:loc").last.text).to eq("https://search.test.datacite.org/")
     end
   end
 end