maltese 0.1.2

data/.travis.yml

@@ -0,0 +1,26 @@
+language: ruby
+rvm:
+- 2.3.3
+
+addons:
+  code_climate:
+    repo_token: "$CODECLIMATE_REPO_TOKEN"
+
+install:
+- travis_retry bundle install
+
+script:
+- bundle exec rspec
+- bundle exec codeclimate-test-reporter
+
+notifications:
+  email: false
+
+deploy:
+  provider: rubygems
+  api_key:
+    secure: uT9YVkRp1usg+glYDSG7KJkm8CQGI8pZDbHlmbPz6ibbA8DVyjbBtYjGbvODCoRkisC24kSy31gMqBSmIxLG0ICv2tOy/iaoiuVeUk6NFfP4dcVGsDueQXjqd6Fjw6fCBg42sojwAVWzvDP2EVjQnbcZqROasLPmKuC2qrm+f9aSYLXmGyBtpvJ5FsfpW33OvE3qJD3y0AlPMdCihPe03FVzSiLNMmGuYOH97MucuWGbUJN+tSFiBfqIrAGT2TQXFrdiT3HtxEt+vNH0cGoLQAKgTgx4XPAcKEjg/cML5yhY/OcPR0uNgqdjxqS3faaH31r1xZaGGfHTf9dj++123YLNHbI8odyA9eF+jYU/3D8UnmMpsTNGZXCFUS8xVUobDcejhPBNhqGPLruLtbvIaqpVZ2bF9BOY1F0ILp4GERzUUUxws+BB1EJ6zFpNrDl7MHlqrc+gRZWcWlazQ82BmLQsTVHiab3ZerGCP4+kYiNeyEnsa3wmVDDd2iffU05Bse44/W1/BKmlzV0QfYl1iMA8lkCrgqmslFecCf0xA01v4CF2Hv63PxOeNmNvZm4VIkgy9uPBjD91AVdscSzCRuTc149OluBqUoxUToX9rEegheUXhWs6ww6DHtlRQI+OBNauRUCo7Fb2zV+gTNzUCSln0fE+z9aLhduuA8JLAKk=
+  gem: maltese
+  on:
+    tags: true
+    repo: datacite/maltese

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## v.0.1 (February 25, 2017)
+
+[maltese 0.1](https://github.com/datacite/maltese/releases/tag/v.0.1) was released on February 25, 2017:
+
+* initial release

data/Dockerfile

@@ -0,0 +1,16 @@
+FROM phusion/passenger-full:0.9.20
+MAINTAINER Martin Fenner "mfenner@datacite.org"
+
+# Install Ruby 2.3.3
+RUN bash -lc 'rvm --default use ruby-2.3.3'
+
+ENV PATH="/usr/local/rvm/gems/ruby-2.3.3/bin:${PATH}"
+
+# Update installed APT packages, clean up APT when done.
+RUN apt-get update && apt-get upgrade -y -o Dpkg::Options::="--force-confold" && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install maltese gem
+RUN /sbin/setuser app gem install maltese
+
+CMD maltese sitemap --sitemap_bucket $SITEMAP_BUCKET --sitemap_url $SITEMAP_URL --from_date $FROM_DATE --until_date $UNTIL_DATE

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,128 @@
+PATH
+  remote: .
+  specs:
+    maltese (0.1.2)
+      activesupport (~> 4.2, >= 4.2.5)
+      fog-aws (~> 0.7.6)
+      maremma (~> 3.5)
+      mime-types (~> 3.1)
+      sitemap_generator (~> 5.1)
+      thor (~> 0.19)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.8)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.5.0)
+      public_suffix (~> 2.0, >= 2.0.2)
+    builder (3.2.3)
+    codeclimate-test-reporter (1.0.6)
+      simplecov
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    docile (1.1.5)
+    excon (0.45.4)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    faraday-encoding (0.0.4)
+      faraday
+    faraday_middleware (0.10.1)
+      faraday (>= 0.7.4, < 1.0)
+    fog-aws (0.7.6)
+      fog-core (~> 1.27)
+      fog-json (~> 1.0)
+      fog-xml (~> 0.1)
+      ipaddress (~> 0.8)
+    fog-core (1.37.0)
+      builder
+      excon (~> 0.45)
+      formatador (~> 0.2)
+    fog-json (1.0.2)
+      fog-core (~> 1.0)
+      multi_json (~> 1.10)
+    fog-xml (0.1.2)
+      fog-core
+      nokogiri (~> 1.5, >= 1.5.11)
+    formatador (0.2.5)
+    hashdiff (0.3.2)
+    i18n (0.8.1)
+    ipaddress (0.8.3)
+    json (2.0.3)
+    maremma (3.5.1)
+      activesupport (~> 4.2, >= 4.2.5)
+      addressable (>= 2.3.6)
+      builder (~> 3.2, >= 3.2.2)
+      excon (~> 0.45.0)
+      faraday (~> 0.9.2)
+      faraday-encoding (~> 0.0.1)
+      faraday_middleware (~> 0.10.0)
+      multi_json (~> 1.11.2)
+      nokogiri (~> 1.6.7)
+      oj (~> 2.18, >= 2.18.1)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    mini_portile2 (2.1.0)
+    minitest (5.10.1)
+    multi_json (1.11.3)
+    multipart-post (2.0.0)
+    nokogiri (1.6.8.1)
+      mini_portile2 (~> 2.1.0)
+    oj (2.18.1)
+    public_suffix (2.0.5)
+    rack (2.0.1)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rake (12.0.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.4)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    safe_yaml (1.0.4)
+    simplecov (0.12.0)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    sitemap_generator (5.3.0)
+      builder (~> 3.0)
+    thor (0.19.4)
+    thread_safe (0.3.6)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    vcr (3.0.3)
+    webmock (1.24.6)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  codeclimate-test-reporter (~> 1.0, >= 1.0.0)
+  maltese!
+  rack-test (~> 0)
+  rake (~> 12.0)
+  rspec (~> 3.4)
+  simplecov (~> 0.12.0)
+  vcr (~> 3.0, >= 3.0.3)
+  webmock (~> 1.22, >= 1.22.3)
+
+BUNDLED WITH
+   1.12.5

data/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 DataCite
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+[![Build Status](https://travis-ci.org/datacite/maltese.svg?branch=master)](https://travis-ci.org/datacite/maltese)
+[![Code Climate](https://codeclimate.com/github/datacite/maltese/badges/gpa.svg)](https://codeclimate.com/github/datacite/maltese)
+[![Test Coverage](https://codeclimate.com/github/datacite/maltese/badges/coverage.svg)](https://codeclimate.com/github/datacite/maltese/coverage)
+
+# Maltese
+
+Ruby gem and command-line tool for generating sitemap files from the DataCite REST API. Uses the [SitemapGenerator](https://github.com/kjvarga/sitemap_generator) gem and can be run as Docker container, e.g. using ECS and triggered by AWS Lambda, as described [here](https://medium.com/@pahud/ecs-task-runner-with-lambda-4594b72ccb#.5xpmf2inz).
+
+Run as a command-line tool:
+
+```
+maltese sitemap --from_date 2017-02-15
+```
+
+## Installation
+
+Requires Ruby 2.2 or later. Then add the following to your `Gemfile` to install the
+latest version:
+
+```ruby
+gem 'maltese'
+```
+
+Then run `bundle install` to install into your environment.
+
+You can also install the gem system-wide in the usual way:
+
+```bash
+gem install maltese
+```
+
+## Development
+
+We use rspec for unit testing:
+
+```
+bundle exec rspec
+```
+
+Follow along via [Github Issues](https://github.com/datacite/toccatore/issues).
+
+### Note on Patches/Pull Requests
+
+* Fork the project
+* Write tests for your new feature or a test that reproduces a bug
+* Implement your feature or make a bug fix
+* Do not mess with Rakefile, version or history
+* Commit, push and make a pull request. Bonus points for topical branches.
+
+## License
+**maltese** is released under the [MIT License](https://github.com/datacite/maltese/blob/master/LICENSE.md).

data/bin/maltese

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("../../lib/maltese", __FILE__)
+
+Maltese::CLI.start

data/lib/maltese/cli.rb

@@ -0,0 +1,30 @@
+# encoding: UTF-8
+
+require "thor"
+require_relative 'sitemap'
+
+module Maltese
+  class CLI < Thor
+    def self.exit_on_failure?
+      true
+    end
+
+    # from http://stackoverflow.com/questions/22809972/adding-a-version-option-to-a-ruby-thor-cli
+    map %w[--version -v] => :__print_version
+
+    desc "--version, -v", "print the version"
+    def __print_version
+      puts Toccatore::VERSION
+    end
+
+    desc "sitemap", "generate sitemap for DataCite Search"
+    method_option :sitemap_bucket, type: :string, default: ENV['SITEMAP_BUCKET']
+    method_option :sitemap_url, type: :string, default: ENV['SITEMAP_URL']
+    method_option :from_date, type: :string, default: (Time.now.to_date - 1.day).iso8601
+    method_option :until_date, type: :string, default: Time.now.to_date.iso8601
+    def sitemap
+      sitemap = Maltese::Sitemap.new(options)
+      sitemap.queue_jobs
+    end
+  end
+end

data/lib/maltese/sitemap.rb

@@ -0,0 +1,140 @@
+module Maltese
+  class Sitemap
+    attr_reader :sitemap_bucket, :sitemap_url, :from_date, :until_date
+
+    def initialize(attributes={})
+      @sitemap_bucket = attributes[:sitemap_bucket]|| "sitemaps.datacite.org"
+      @sitemap_url = attributes[:sitemap_url] || "https://search.datacite.org"
+      @from_date = attributes[:from_date].presence || (Time.now.to_date - 1.day).iso8601
+      @until_date = attributes[:until_date].presence || Time.now.to_date.iso8601
+    end
+
+    # load ENV variables from container environment if json file exists
+    # see https://github.com/phusion/baseimage-docker#envvar_dumps
+    env_json_file = "/etc/container_environment.json"
+    if File.size?(env_json_file).to_i > 2
+      env_vars = JSON.parse(File.read(env_json_file))
+      env_vars.each { |k, v| ENV[k] = v }
+    end
+
+    def search_path
+      "#{sitemap_url}/api?"
+    end
+
+    def sitemaps_host
+      "http://#{sitemap_bucket}.s3.amazonaws.com/"
+    end
+
+    def sitemaps_path
+      'sitemaps/'
+    end
+
+    def timeout
+      120
+    end
+
+    def job_batch_size
+      50000
+    end
+
+    def sitemap
+      @sitemap ||= SitemapGenerator::LinkSet.new(
+        default_host: sitemap_url,
+        adapter: s3_adapter,
+        sitemaps_host: sitemaps_host,
+        sitemaps_path: sitemaps_path,
+        finalize: false)
+    end
+
+    def s3_adapter
+      SitemapGenerator::S3Adapter.new(fog_provider: 'AWS',
+                                      aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+                                      aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
+                                      fog_directory: sitemap_bucket,
+                                      fog_region: ENV['AWS_REGION'])
+    end
+
+    def fog_storage
+      Fog::Storage.new(provider: 'AWS',
+                       aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+                       aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'])
+    end
+
+    def queue_jobs(options={})
+      total = get_total(options)
+
+      if total > 0
+        puts process_data(options.merge(total: total))
+      else
+        puts "No works found for date range #{from_date} - #{until_date}."
+      end
+
+      # return number of works queued
+      total
+    end
+
+    def get_total(options={})
+      query_url = get_query_url(options.merge(rows: 0))
+      result = Maremma.get(query_url, options)
+      result.body.fetch("data", {}).fetch("response", {}).fetch("numFound", 0)
+    end
+
+    def get_query_url(options={})
+      options[:offset] = options[:offset].to_i || 0
+      options[:rows] = options[:rows].presence || job_batch_size
+
+      updated = "updated:[#{from_date}T00:00:00Z TO #{until_date}T23:59:59Z]"
+      fq = "#{updated} AND has_metadata:true AND is_active:true"
+
+      params = { q: "*:*",
+                 fq: fq,
+                 start: options[:offset],
+                 rows: options[:rows],
+                 fl: "doi,updated",
+                 sort: "updated asc",
+                 wt: "json" }
+      search_path + URI.encode_www_form(params)
+    end
+
+    def process_data(options = {})
+      options[:start_time] = Time.now
+
+      # walk through paginated results
+      total_pages = (options[:total].to_f / job_batch_size).ceil
+
+      (0...total_pages).each do |page|
+        options[:offset] = page * job_batch_size
+        data = get_data(options.merge(timeout: timeout))
+        parse_data(data)
+      end
+
+      push_data(options)
+    end
+
+    def get_data(options={})
+      query_url = get_query_url(options)
+      Maremma.get(query_url, options)
+    end
+
+    def parse_data(result, options={})
+      return result.body.fetch("errors") if result.body.fetch("errors", nil).present?
+
+      items = result.body.fetch("data", {}).fetch('response', {}).fetch('docs', nil)
+      Array(items).each do |item|
+        loc = "/works/" + item.fetch("doi")
+        sitemap.add loc, changefreq: "monthly", lastmod: item.fetch("updated")
+      end
+      sitemap.sitemap.link_count
+    end
+
+    def push_data(options={})
+      # sync time with AWS S3 before uploading
+      fog_storage.sync_clock
+
+      sitemap.finalize!
+      options[:start_time] ||= Time.now
+      sitemap.sitemap_index.stats_summary(:time_taken => Time.now - options[:start_time])
+      sitemap.sitemap.link_count
+    end
+  end
+end

data/lib/maltese/utils.rb

@@ -0,0 +1,87 @@
+module Maltese
+  module Utils
+    # load ENV variables from container environment if json file exists
+    # see https://github.com/phusion/baseimage-docker#envvar_dumps
+    env_json_file = "/etc/container_environment.json"
+    if File.size?(env_json_file).to_i > 2
+      env_vars = JSON.parse(File.read(env_json_file))
+      env_vars.each { |k, v| ENV[k] = v }
+    end
+
+    def queue_jobs(options={})
+      options[:offset] = options[:offset].to_i || 0
+      options[:rows] = options[:rows].presence || job_batch_size
+
+      total = get_total(options)
+
+      if total > 0
+        puts process_data(options.merge(total: total))
+      else
+        puts "No works found for date range #{from_date} - #{until_date}."
+      end
+
+      # return number of works queued
+      total
+    end
+
+    def get_total(options={})
+      query_url = get_query_url(options.merge(rows: 0))
+      result = Maremma.get(query_url, options)
+      result.body.fetch("data", {}).fetch("response", {}).fetch("numFound", 0)
+    end
+
+    def get_query_url(options={})
+      updated = "updated:[#{from_date}T00:00:00Z TO #{until_date}T23:59:59Z]"
+      fq = "#{updated} AND has_metadata:true AND is_active:true"
+
+      params = { q: "*:*",
+                 fq: fq,
+                 start: options[:offset],
+                 rows: options[:rows],
+                 fl: "doi,updated",
+                 sort: "updated asc",
+                 wt: "json" }
+      url +  URI.encode_www_form(params)
+    end
+
+    def process_data(options = {})
+      options[:start_time] = Time.now
+
+      # walk through paginated results
+      total_pages = (options[:total].to_f / job_batch_size).ceil
+
+      (0...total_pages).each do |page|
+        options[:offset] = page * job_batch_size
+        data = get_data(options.merge(timeout: timeout))
+        parse_data(data)
+      end
+
+      push_data(options)
+    end
+
+    def get_data(options={})
+      query_url = get_query_url(options)
+      Maremma.get(query_url, options)
+    end
+
+    def parse_data(result, options={})
+      return result.body.fetch("errors") if result.body.fetch("errors", nil).present?
+
+      items = result.body.fetch("data", {}).fetch('response', {}).fetch('docs', nil)
+      Array(items).each do |item|
+        loc = "/works/" + item.fetch("doi")
+        sitemap.add loc, changefreq: "monthly", lastmod: item.fetch("updated")
+      end
+      sitemap.sitemap.link_count
+    end
+
+    def push_data(options={})
+      # sync time with AWS S3 before uploading
+      fog_storage.sync_clock
+
+      sitemap.finalize!
+      time_taken = Time.now - options[:start_time]
+      sitemap.sitemap_index.stats_summary(:time_taken => time_taken)
+    end
+  end
+end

data/lib/maltese/version.rb

@@ -0,0 +1,3 @@
+module Maltese
+  VERSION = "0.1.2"
+end

data/lib/maltese.rb

@@ -0,0 +1,8 @@
+require 'sitemap_generator'
+require 'maremma'
+require 'fog/aws'
+require 'mime/types'
+require 'active_support/all'
+
+require "maltese/sitemap"
+require "maltese/cli"

data/maltese.gemspec

@@ -0,0 +1,37 @@
+require "date"
+require File.expand_path("../lib/maltese/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.authors       = "Martin Fenner"
+  s.email         = "mfenner@datacite.org"
+  s.name          = "maltese"
+  s.homepage      = "https://github.com/datacite/maltese"
+  s.summary       = "Ruby library to generate sitemap for DataCite Search"
+  s.date          = Date.today
+  s.description   = "Ruby library to generate sitemap for DataCite Search."
+  s.require_paths = ["lib"]
+  s.version       = Maltese::VERSION
+  s.extra_rdoc_files = ["README.md"]
+  s.license       = 'MIT'
+
+  # Declary dependencies here, rather than in the Gemfile
+  s.add_dependency 'maremma', '~> 3.5'
+  s.add_dependency 'activesupport', '~> 4.2', '>= 4.2.5'
+  s.add_dependency 'thor', '~> 0.19'
+  s.add_dependency 'sitemap_generator', '~> 5.1'
+  s.add_dependency 'fog-aws', '~> 0.7.6'
+  s.add_dependency 'mime-types', '~> 3.1'
+  s.add_development_dependency 'bundler', '~> 1.0'
+  s.add_development_dependency 'rspec', '~> 3.4'
+  s.add_development_dependency 'rake', '~> 12.0'
+  s.add_development_dependency 'rack-test', '~> 0'
+  s.add_development_dependency 'vcr', '~> 3.0', '>= 3.0.3'
+  s.add_development_dependency 'webmock', '~> 1.22', '>= 1.22.3'
+  s.add_development_dependency 'codeclimate-test-reporter', '~> 1.0', '>= 1.0.0'
+  s.add_development_dependency 'simplecov', '~> 0.12.0'
+
+  s.require_paths = ["lib"]
+  s.files       = `git ls-files`.split($/)
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables = ["maltese"]
+end

data/spec/cli_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+require 'maltese/cli'
+
+describe Maltese::CLI do
+  let(:subject) do
+    described_class.new
+  end
+
+  let(:push_url) { ENV['VOLPINO_URL'] }
+  let(:access_token) { ENV['VOLPINO_TOKEN'] }
+  let(:from_date) { "2015-04-07" }
+  let(:until_date) { "2015-04-08" }
+  let(:cli_options) { { push_url: push_url,
+                        access_token: access_token,
+                        from_date: from_date,
+                        until_date: until_date } }
+
+  describe "sitemap", vcr: true, :order => :defined do
+    it 'should succeed' do
+      subject.options = cli_options
+      expect { subject.sitemap }.to output(/2522 links/).to_stdout
+      sitemap = Zlib::GzipReader.open("public/sitemap.xml.gz") { |gz| gz.read }
+      doc = Nokogiri::XML(sitemap)
+      expect(doc.xpath("//xmlns:url").size).to eq(2522)
+      expect(doc.xpath("//xmlns:loc").last.text).to eq("https://search.datacite.org/works/10.6084/M9.FIGSHARE.1371139")
+    end
+
+    it 'should succeed with no works' do
+      from_date = "2005-04-07"
+      until_date = "2005-04-08"
+      subject.options = { push_url: push_url,
+                          access_token: access_token,
+                          from_date: from_date,
+                          until_date: until_date }
+      expect { subject.sitemap }.to output("No works found for date range 2005-04-07 - 2005-04-08.\n").to_stdout
+    end
+
+    # it 'should fail' do
+    #   subject.options = cli_options.except(:access_token)
+    #   expect { subject.sitemap }.to output(/An error occured: Access token missing.\n/).to_stdout
+    # end
+  end
+end