maltese 0.1.2

data/spec/sitemap_spec.rb

@@ -0,0 +1,111 @@
+require 'spec_helper'
+
+describe Maltese::Sitemap, vcr: true do
+  before(:each) { allow(Time).to receive(:now).and_return(Time.mktime(2015, 4, 8)) }
+
+  subject { Maltese::Sitemap.new }
+
+  context "get_query_url" do
+    it "default" do
+      expect(subject.get_query_url).to eq("https://search.datacite.org/api?q=*%3A*&fq=updated%3A%5B2015-04-07T00%3A00%3A00Z+TO+2015-04-08T23%3A59%3A59Z%5D+AND+has_metadata%3Atrue+AND+is_active%3Atrue&start=0&rows=50000&fl=doi%2Cupdated&sort=updated+asc&wt=json")
+    end
+
+    it "with zero rows" do
+      expect(subject.get_query_url(rows: 0)).to eq("https://search.datacite.org/api?q=*%3A*&fq=updated%3A%5B2015-04-07T00%3A00%3A00Z+TO+2015-04-08T23%3A59%3A59Z%5D+AND+has_metadata%3Atrue+AND+is_active%3Atrue&start=0&rows=0&fl=doi%2Cupdated&sort=updated+asc&wt=json")
+    end
+
+    it "with different from_date and until_date" do
+      subject = Maltese::Sitemap.new(from_date: "2015-04-05", until_date: "2015-04-05")
+      expect(subject.get_query_url).to eq("https://search.datacite.org/api?q=*%3A*&fq=updated%3A%5B2015-04-05T00%3A00%3A00Z+TO+2015-04-05T23%3A59%3A59Z%5D+AND+has_metadata%3Atrue+AND+is_active%3Atrue&start=0&rows=50000&fl=doi%2Cupdated&sort=updated+asc&wt=json")
+    end
+
+    it "with offset" do
+      expect(subject.get_query_url(offset: 250)).to eq("https://search.datacite.org/api?q=*%3A*&fq=updated%3A%5B2015-04-07T00%3A00%3A00Z+TO+2015-04-08T23%3A59%3A59Z%5D+AND+has_metadata%3Atrue+AND+is_active%3Atrue&start=250&rows=50000&fl=doi%2Cupdated&sort=updated+asc&wt=json")
+    end
+
+    it "with rows" do
+      expect(subject.get_query_url(rows: 250)).to eq("https://search.datacite.org/api?q=*%3A*&fq=updated%3A%5B2015-04-07T00%3A00%3A00Z+TO+2015-04-08T23%3A59%3A59Z%5D+AND+has_metadata%3Atrue+AND+is_active%3Atrue&start=0&rows=250&fl=doi%2Cupdated&sort=updated+asc&wt=json")
+    end
+  end
+
+  context "get_total" do
+    it "with works" do
+      expect(subject.get_total).to eq(2521)
+    end
+
+    it "with no works" do
+      subject = Maltese::Sitemap.new(from_date: "2005-04-07", until_date: "2005-04-08")
+      expect(subject.get_total).to eq(0)
+    end
+  end
+
+  context "queue_jobs" do
+    it "should report if there are no works returned by the Datacite Solr API" do
+      subject = Maltese::Sitemap.new(from_date: "2005-04-07", until_date: "2005-04-08")
+      response = subject.queue_jobs
+      expect(response).to eq(0)
+    end
+
+    it "should report if there are works returned by the Datacite Solr API" do
+      response = subject.queue_jobs
+      expect(response).to eq(2521)
+    end
+  end
+
+  context "get_data" do
+    it "should report if there are no works returned by the Datacite Solr API" do
+      subject = Maltese::Sitemap.new(from_date: "2005-04-07", until_date: "2005-04-08")
+      response = subject.get_data
+      expect(response.body["data"]["response"]["numFound"]).to eq(0)
+    end
+
+    it "should report if there are works returned by the Datacite Solr API" do
+      response = subject.get_data
+      expect(response.body["data"]["response"]["numFound"]).to eq(2521)
+      doc = response.body["data"]["response"]["docs"].first
+      expect(doc["doi"]).to eq("10.15468/DL.6OKWWI")
+    end
+
+    it "should catch errors with the Datacite Solr API" do
+      stub = stub_request(:get, subject.get_query_url(rows: 0)).to_return(:status => [408])
+      response = subject.get_data(rows: 0)
+      expect(response.body).to eq("errors"=>[{"status"=>408, "title"=>"Request timeout"}])
+      expect(stub).to have_been_requested
+    end
+  end
+
+  context "parse_data" do
+    it "should report if there are no works returned by the Datacite Solr API" do
+      body = File.read(fixture_path + 'sitemap_nil.json')
+      result = OpenStruct.new(body: { "data" => JSON.parse(body) })
+      expect(subject.parse_data(result)).to eq(0)
+    end
+
+    it "should report if there are works returned by the Datacite Solr API" do
+      body = File.read(fixture_path + 'sitemap.json')
+      result = OpenStruct.new(body: { "data" => JSON.parse(body) })
+      response = subject.parse_data(result)
+      expect(response).to eq(2522)
+    end
+
+    it "should catch timeout errors with the Datacite Solr API" do
+      result = OpenStruct.new(body: { "errors" => [{ "title" => "the server responded with status 408 for https://search.datacite.org", "status" => 408 }] })
+      response = subject.parse_data(result)
+      expect(response).to eq(result.body["errors"])
+    end
+  end
+
+  context "push_data" do
+    it "should report if there are no works returned by the Datacite Solr API" do
+      result = []
+      expect { subject.push_data }.to output(/1 links/).to_stdout
+    end
+
+    it "should report if there are works returned by the Datacite Solr API" do
+      body = File.read(fixture_path + 'sitemap.json')
+      result = OpenStruct.new(body: { "data" => JSON.parse(body) })
+      result = subject.parse_data(result)
+      expect { subject.push_data }.to output(/2522 links/).to_stdout
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,95 @@
+ENV["FOG_DIRECTORY"] = "sitemaps.test.datacite.org"
+
+require 'bundler/setup'
+Bundler.setup
+
+require 'simplecov'
+SimpleCov.start
+
+require 'maltese'
+require 'rspec'
+require 'rack/test'
+require 'webmock/rspec'
+require 'nokogiri'
+require 'vcr'
+
+RSpec.configure do |config|
+  config.order = :random
+  config.include WebMock::API
+  config.include Rack::Test::Methods
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.allow_message_expectations_on_nil = false
+  end
+
+  config.before do
+    ARGV.replace []
+  end
+end
+
+def fixture_path
+  File.expand_path("../fixtures", __FILE__) + '/'
+end
+
+# This code was adapted from Thor, available under MIT-LICENSE
+# Copyright (c) 2008 Yehuda Katz, Eric Hodel, et al.
+def capture(stream)
+  begin
+    stream = stream.to_s
+    eval "$#{stream} = StringIO.new"
+    yield
+    result = eval("$#{stream}").string
+  ensure
+    eval("$#{stream} = #{stream.upcase}")
+  end
+
+  result
+end
+
+def capture_stdout(&block)
+  original_stdout = $stdout
+  $stdout = fake = StringIO.new
+  begin
+    yield
+  ensure
+    $stdout = original_stdout
+  end
+  fake.string
+end
+
+def capture_stderr(&block)
+  original_stderr = $stderr
+  $stderr = fake = StringIO.new
+  begin
+    yield
+  ensure
+    $stderr = original_stderr
+  end
+  fake.string
+end
+
+# This code was adapted from Ruby on Rails, available under MIT-LICENSE
+# Copyright (c) 2004-2013 David Heinemeier Hansson
+def silence_warnings
+  old_verbose, $VERBOSE = $VERBOSE, nil
+  yield
+ensure
+  $VERBOSE = old_verbose
+end
+
+alias silence capture
+
+VCR.configure do |c|
+  c.cassette_library_dir = "spec/fixtures/vcr_cassettes"
+  c.hook_into :webmock
+  c.ignore_localhost = true
+  c.ignore_hosts 'codeclimate.com'
+  c.configure_rspec_metadata!
+  c.filter_sensitive_data("<AWS_ACCESS_KEY_ID>") { ENV["AWS_ACCESS_KEY_ID"] }
+  c.filter_sensitive_data("<AWS_SECRET_ACCESS_KEY>") { ENV["AWS_SECRET_ACCESS_KEY"] }
+  c.filter_sensitive_data("<FOG_DIRECTORY>") { ENV['FOG_DIRECTORY'] }
+  c.filter_sensitive_data("<FOG_REGION>") { ENV["FOG_REGION"] }
+end

metadata

@@ -0,0 +1,310 @@
+--- !ruby/object:Gem::Specification
+name: maltese
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Martin Fenner
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: maremma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.5
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: sitemap_generator
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: fog-aws
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.6
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.3
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.22'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.22.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.22'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.22.3
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+description: Ruby library to generate sitemap for DataCite Search.
+email: mfenner@datacite.org
+executables:
+- maltese
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- ".codeclimate.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- CHANGELOG.md
+- Dockerfile
+- Gemfile
+- Gemfile.lock
+- LICENSE.md
+- README.md
+- bin/maltese
+- lib/maltese.rb
+- lib/maltese/cli.rb
+- lib/maltese/sitemap.rb
+- lib/maltese/utils.rb
+- lib/maltese/version.rb
+- maltese.gemspec
+- public/sitemap.xml.gz
+- spec/cli_spec.rb
+- spec/fixtures/sitemap.json
+- spec/fixtures/sitemap_nil.json
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_data/should_report_if_there_are_no_works_returned_by_the_Datacite_Solr_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_data/should_report_if_there_are_works_returned_by_the_Datacite_Solr_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_total/with_no_works.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_total/with_works.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_no_works_returned_by_the_Datacite_Solr_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_works_returned_by_the_Datacite_Solr_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/queue_jobs/should_report_if_there_are_no_works_returned_by_the_Datacite_Solr_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/queue_jobs/should_report_if_there_are_works_returned_by_the_Datacite_Solr_API.yml
+- spec/sitemap_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/datacite/maltese
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Ruby library to generate sitemap for DataCite Search
+test_files:
+- spec/cli_spec.rb
+- spec/fixtures/sitemap.json
+- spec/fixtures/sitemap_nil.json
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_data/should_report_if_there_are_no_works_returned_by_the_Datacite_Solr_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_data/should_report_if_there_are_works_returned_by_the_Datacite_Solr_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_total/with_no_works.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_total/with_works.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_no_works_returned_by_the_Datacite_Solr_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_works_returned_by_the_Datacite_Solr_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/queue_jobs/should_report_if_there_are_no_works_returned_by_the_Datacite_Solr_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/queue_jobs/should_report_if_there_are_works_returned_by_the_Datacite_Solr_API.yml
+- spec/sitemap_spec.rb
+- spec/spec_helper.rb
+has_rdoc: