govuk_seed_crawler 2.1.0 → 3.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b0d80cc9a1bf29784700e23b1f8c883fc6012a2e7a47776cc3af04ca87f253e7
+  data.tar.gz: ca2d04361ff8e9d9b3cf34f0432eb786b026ffcfd035b1a5faeed6c32801880a
+SHA512:
+  metadata.gz: 895569f56489bbfce199620c211742b226b56d0cc8a03d17451b8a382f2743ba1d7c7821ff9fe75925d411d9b99a595981cf72fb5334907f59f33bf0a24ec000
+  data.tar.gz: 8ea165b2ed3d62a63a5a331b03e14e2fe0dd68713bf7732d28bd2b5cfc4e20a6b5a4932c503fc0514d1a0ea1c6d58343afe864612d8190a2521638d62eea0e83

data/.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: bundler
+    directory: /
+    schedule:
+      interval: daily
+  - package-ecosystem: "github-actions"
+    directory: /
+    schedule:
+      interval: daily

data/.github/workflows/ci.yml

@@ -0,0 +1,47 @@
+on: [push, pull_request]
+
+jobs:
+  # This matrix job runs the test suite against multiple Ruby versions
+  test_matrix:
+    strategy:
+      fail-fast: false
+      matrix:
+        # Due to https://github.com/actions/runner/issues/849, we have to use quotes for '3.0'
+        ruby: [ 2.7, '3.0', 3.1 ]
+    runs-on: ubuntu-latest
+    services:
+      rabbitmq:
+        image: rabbitmq
+        env:
+          RABBITMQ_DEFAULT_USER: rabbitmq
+          RABBITMQ_DEFAULT_PASS: rabbitmq
+        ports:
+          - 5672:5672
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - run: bundle exec rake
+        env:
+          AMQP_USER: rabbitmq
+          AMQP_PASS: rabbitmq
+
+  # Branch protection rules cannot directly depend on status checks from matrix jobs.
+  # So instead we define `test` as a dummy job which only runs after the preceding `test_matrix` checks have passed.
+  # Solution inspired by: https://github.community/t/status-check-for-a-matrix-jobs/127354/3
+  test:
+    needs: test_matrix
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "All matrix tests have passed 🚀"
+
+  publish:
+    needs: test
+    if: ${{ github.ref == 'refs/heads/main' }}
+    permissions:
+      contents: write
+    uses: alphagov/govuk-infrastructure/.github/workflows/publish-rubygem.yaml@main
+    secrets:
+      GEM_HOST_API_KEY: ${{ secrets.ALPHAGOV_RUBYGEMS_API_KEY }}

data/.rubocop.yml

@@ -0,0 +1,19 @@
+inherit_gem:
+  rubocop-govuk:
+    - config/default.yml
+    - config/rake.yml
+    - config/rspec.yml
+
+inherit_mode:
+  merge:
+    - Exclude
+
+# **************************************************************
+# TRY NOT TO ADD OVERRIDES IN THIS FILE
+#
+# This repo is configured to follow the RuboCop GOV.UK styleguide.
+# Any rules you override here will cause this repo to diverge from
+# the way we write code in all other GOV.UK repos.
+#
+# See https://github.com/alphagov/rubocop-govuk/blob/main/CONTRIBUTING.md
+# **************************************************************

data/.ruby-version

@@ -1 +1 @@
-1.9.3-p550
+2.7.6

data/CHANGELOG.md

@@ -0,0 +1,11 @@
+# Changelog
+
+- We use the [GOV.UK versioning guidelines](https://docs.publishing.service.gov.uk/manual/publishing-a-ruby-gem.html#versioning).
+- Mark breaking changes with `BREAKING:`. Be sure to include instructions on how applications should be upgraded.
+- Don't include changes that are purely internal. The CHANGELOG should be a
+  useful summary for people upgrading their application, not a replication
+  of the commit log.
+
+## Unreleased
+
+- Drop support for Ruby < 2.7

data/Gemfile

@@ -1,4 +1,4 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
 # Specify your gem's dependencies in govuk_seed_crawler.gemspec
 gemspec

data/{LICENSE.txt → LICENCE}

@@ -1,6 +1,6 @@
-(c) 2014 Crown copyright
+The MIT License (MIT)
 
-MIT License
+Copyright (C) 2014 Crown Copyright (Government Digital Service)
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -38,3 +38,7 @@ bundle exec seed-crawler --help
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
+
+## Licence
+
+[MIT License](LICENCE)

data/Rakefile

@@ -1,11 +1,7 @@
-require 'rspec/core/rake_task'
+require "rspec/core/rake_task"
+require "rubocop/rake_task"
 
-RSpec::Core::RakeTask.new(:spec) do |task|
-  task.pattern = FileList['spec/govuk_seed_crawler/**/*_spec.rb']
-end
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
 
-RSpec::Core::RakeTask.new(:integration) do |task|
-  task.pattern = FileList['spec/integration/**/*_spec.rb']
-end
-
-task :default => :spec
+task default: %i[rubocop spec]

data/govuk_seed_crawler.gemspec

@@ -1,35 +1,36 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'govuk_seed_crawler/version'
+require "govuk_seed_crawler/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "govuk_seed_crawler"
   spec.version       = GovukSeedCrawler::VERSION
-  spec.authors       = ['GOV.UK developers']
+  spec.authors       = ["GOV.UK developers"]
   spec.email         = ["govuk-dev@digital.cabinet-office.gov.uk"]
-  spec.summary       = %q{Retrieves a list of URLs to seed the crawler by publishing them to a RabbitMQ exchange.}
+  spec.summary       = "Retrieves a list of URLs to seed the crawler by publishing them to a RabbitMQ exchange."
   spec.homepage      = "https://github.com/alphagov/govuk_seed_crawler"
   spec.license       = "MIT"
 
+  spec.required_ruby_version = ">= 2.7"
+
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = %w[lib]
 
-  spec.add_runtime_dependency "bunny", "~> 1.3"
-  spec.add_runtime_dependency "crack", "0.4.4"
-  spec.add_runtime_dependency "ffi", "1.10.0"
-  spec.add_runtime_dependency "nokogiri", "~> 1.6.0"
+  spec.add_runtime_dependency "bunny", ">= 1.3", "< 3.0"
+  spec.add_runtime_dependency "crack", "0.4.5"
+  spec.add_runtime_dependency "nokogiri", ">= 1.6", "< 1.14"
   # Something, somewhere, sometimes requires public_suffix.
   # public_suffix > 1.5 requires ruby > 2.
-  spec.add_runtime_dependency "public_suffix", "~> 1.4.6"
-  spec.add_runtime_dependency "sitemap-parser", "~> 0.3.0"
+  spec.add_runtime_dependency "public_suffix", ">= 1.4.6", "< 5.1.0"
+  spec.add_runtime_dependency "sitemap-parser", ">= 0.3", "< 0.6"
   spec.add_runtime_dependency "slop", "~> 3.6.0"
 
   spec.add_development_dependency "pry"
-  spec.add_development_dependency "rake", "~> 0.9"
+  spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-mocks", "~> 3.0"
-  spec.add_development_dependency "webmock", "~> 1.18.0"
+  spec.add_development_dependency "rubocop-govuk", "4.8.0"
+  spec.add_development_dependency "webmock", "~> 3.18"
 end

data/lib/govuk_seed_crawler/amqp_client.rb

@@ -1,4 +1,4 @@
-require 'bunny'
+require "bunny"
 
 module GovukSeedCrawler
   class AmqpClient
@@ -21,8 +21,8 @@ module GovukSeedCrawler
 
       GovukSeedCrawler.logger.debug("Publishing '#{body}' to topic '#{topic}'")
 
-      @channel.topic(exchange, :durable => true)
-        .publish(body, :routing_key => topic)
+      @channel.topic(exchange, durable: true)
+        .publish(body, routing_key: topic)
     end
   end
 end

data/lib/govuk_seed_crawler/cli_parser.rb

@@ -1,4 +1,4 @@
-require 'slop'
+require "slop"
 
 module GovukSeedCrawler
   class CLIException < StandardError
@@ -12,17 +12,17 @@ module GovukSeedCrawler
 
   class CLIParser
     DEFAULTS = {
-      :exchange => "govuk_crawler_exchange",
-      :help => nil,
-      :host => "localhost",
-      :password => "guest",
-      :port => "5672",
-      :quiet => false,
-      :topic => "#",
-      :username => "guest",
-      :verbose => false,
-      :version => nil,
-      :vhost => "/"
+      exchange: "govuk_crawler_exchange",
+      help: nil,
+      host: "localhost",
+      password: "guest",
+      port: "5672",
+      quiet: false,
+      topic: "#",
+      username: "guest",
+      verbose: false,
+      version: nil,
+      vhost: "/",
     }.freeze
 
     ENV_AMQP_PASS_KEY = "GOVUK_CRAWLER_AMQP_PASS".freeze
@@ -32,19 +32,19 @@ module GovukSeedCrawler
     end
 
     def options
-      Slop.parse!(@argv_array, :help => true) do
-        banner <<-EOS
-Usage: #{$PROGRAM_NAME} site_root [options]
+      Slop.parse!(@argv_array, help: true) do
+        banner <<~HELP
+          Usage: #{$PROGRAM_NAME} site_root [options]
 
-Seeds an AMQP topic exchange with messages, each containing a URL, for the GOV.UK Crawler Worker
-to consume:
+          Seeds an AMQP topic exchange with messages, each containing a URL, for the GOV.UK Crawler Worker
+          to consume:
 
-https://github.com/alphagov/govuk_crawler_worker
+          https://github.com/alphagov/govuk_crawler_worker
 
-The AMQP password can also be set as an environment variable and will be read from
-`#{ENV_AMQP_PASS_KEY}`. If both the environment variable and command-line option for password
-are set, the environment variable will take higher precedent.
-        EOS
+          The AMQP password can also be set as an environment variable and will be read from
+          `#{ENV_AMQP_PASS_KEY}`. If both the environment variable and command-line option for password
+          are set, the environment variable will take higher precedent.
+        HELP
 
         on :version, "Display version and exit" do
           puts "Version: #{GovukSeedCrawler::VERSION}"
@@ -75,7 +75,7 @@ are set, the environment variable will take higher precedent.
       options_hash = opts.to_hash
       options_hash[:password] = ENV[ENV_AMQP_PASS_KEY] unless ENV[ENV_AMQP_PASS_KEY].nil?
 
-      return options_hash, @argv_array.first
+      [options_hash, @argv_array.first]
     end
   end
 end

data/lib/govuk_seed_crawler/cli_runner.rb

@@ -13,10 +13,10 @@ module GovukSeedCrawler
     end
 
     def run
-      Seeder::seed(@site_root, @options)
+      Seeder.seed(@site_root, @options)
     end
 
-    private
+  private
 
     def set_logging_level(cli_options)
       if cli_options[:verbose]

data/lib/govuk_seed_crawler/indexer.rb

@@ -1,4 +1,4 @@
-require 'sitemap-parser'
+require "sitemap-parser"
 
 module GovukSeedCrawler
   class Indexer
@@ -9,7 +9,7 @@ module GovukSeedCrawler
 
       GovukSeedCrawler.logger.info("Retrieving list of URLs for #{site_root}")
 
-      sitemap = SitemapParser.new("#{site_root}/sitemap.xml", {recurse: true})
+      sitemap = SitemapParser.new("#{site_root}/sitemap.xml", { recurse: true })
       @urls = sitemap.to_a
 
       GovukSeedCrawler.logger.info("Found #{@urls.count} URLs")

data/lib/govuk_seed_crawler/version.rb

@@ -1,3 +1,3 @@
 module GovukSeedCrawler
-  VERSION = "2.1.0"
+  VERSION = "3.1.0".freeze
 end

data/lib/govuk_seed_crawler.rb

@@ -1,17 +1,21 @@
-require 'govuk_seed_crawler/amqp_client'
-require 'govuk_seed_crawler/cli_parser'
-require 'govuk_seed_crawler/cli_runner'
-require 'govuk_seed_crawler/indexer'
-require 'govuk_seed_crawler/seeder'
-require 'govuk_seed_crawler/version'
+require "govuk_seed_crawler/amqp_client"
+require "govuk_seed_crawler/cli_parser"
+require "govuk_seed_crawler/cli_runner"
+require "govuk_seed_crawler/indexer"
+require "govuk_seed_crawler/seeder"
+require "govuk_seed_crawler/version"
 
 module GovukSeedCrawler
-  def self.logger
-    unless @logger
-      @logger = Logger.new(STDOUT)
-      @logger.level = Logger::INFO
-    end
+  class << self
+    attr_writer :logger
+
+    def logger
+      unless @logger
+        @logger = Logger.new($stdout)
+        @logger.level = Logger::INFO
+      end
 
-    @logger
+      @logger
+    end
   end
 end

data/spec/govuk_seed_crawler/amqp_client_spec.rb

@@ -1,55 +1,54 @@
-require 'spec_helper'
-
 describe GovukSeedCrawler::AmqpClient do
   let(:exchange) { "govuk_seed_crawler_spec_exchange" }
-  let(:options) {{
-    :host => ENV.fetch("AMQP_HOST", "localhost"),
-    :user => ENV.fetch("AMQP_USER", "govuk_seed_crawler"),
-    :pass => ENV.fetch("AMQP_PASS", "govuk_seed_crawler"),
-  }}
-  subject { GovukSeedCrawler::AmqpClient.new(options) }
+  let(:options) do
+    {
+      host: ENV.fetch("AMQP_HOST", "localhost"),
+      user: ENV.fetch("AMQP_USER", "govuk_seed_crawler"),
+      pass: ENV.fetch("AMQP_PASS", "govuk_seed_crawler"),
+    }
+  end
 
   it "responds to #channel" do
-    expect(subject).to respond_to(:channel)
+    expect(described_class.new(options)).to respond_to(:channel)
   end
 
   it "responds to #close" do
-    expect(subject).to respond_to(:close)
+    expect(described_class.new(options)).to respond_to(:close)
   end
 
   it "closes the connection to the AMQP server" do
-    mock_bunny = double(:mock_bunny,
-      :start => true, :create_channel => true, :close => true)
+    mock_bunny = instance_double(Bunny::Session,
+                                 start: true,
+                                 create_channel: true,
+                                 close: true)
     allow(Bunny).to receive(:new).and_return(mock_bunny)
     expect(mock_bunny).to receive(:close).once
 
-    subject.close
+    described_class.new(options).close
   end
 
-  context "#publish" do
-    context "error handling" do
-      it "raises an exception if exchange is nil" do
-        expect {
-          subject.publish(nil, "#", "some body")
-        }.to raise_exception(RuntimeError, "Exchange cannot be nil")
-      end
-
-      it "raises an exception if topic is nil" do
-        expect {
-          subject.publish(exchange, nil, "some body")
-        }.to raise_exception(RuntimeError, "Topic cannot be nil")
-      end
-
-      it "raises an exception if body is nil" do
-        expect {
-          subject.publish(exchange, "#", nil)
-        }.to raise_exception(RuntimeError, "Message body cannot be nil")
-      end
+  describe "#publish" do
+    it "raises an exception if exchange is nil" do
+      expect {
+        described_class.new(options).publish(nil, "#", "some body")
+      }.to raise_exception(RuntimeError, "Exchange cannot be nil")
+    end
+
+    it "raises an exception if topic is nil" do
+      expect {
+        described_class.new(options).publish(exchange, nil, "some body")
+      }.to raise_exception(RuntimeError, "Topic cannot be nil")
+    end
+
+    it "raises an exception if body is nil" do
+      expect {
+        described_class.new(options).publish(exchange, "#", nil)
+      }.to raise_exception(RuntimeError, "Message body cannot be nil")
     end
 
     it "allows publishing against an exchange" do
-      expect(subject.publish(exchange, "#", "some body"))
-        .to_not be_nil
+      expect(described_class.new(options).publish(exchange, "#", "some body"))
+        .not_to be_nil
     end
   end
 end

data/spec/govuk_seed_crawler/cli_parser_spec.rb

@@ -1,28 +1,26 @@
-require 'spec_helper'
-
 describe GovukSeedCrawler::CLIParser do
   it "requires the site_root to be provided" do
     expect {
-      GovukSeedCrawler::CLIParser.new([]).parse
+      described_class.new([]).parse
     }.to raise_exception(GovukSeedCrawler::CLIException, "site_root must be provided")
   end
 
   it "provides the defaults when just given the site_root" do
-    options, site_root = GovukSeedCrawler::CLIParser.new(["https://www.example.com"]).parse
+    options, site_root = described_class.new(["https://www.example.com"]).parse
 
     expect(options).to eq(GovukSeedCrawler::CLIParser::DEFAULTS)
     expect(site_root).to eq("https://www.example.com")
   end
 
-  it "should tell us when we've given too many arguments" do
+  it "tells us when we've given too many arguments" do
     expect {
-      GovukSeedCrawler::CLIParser.new(["a", "b"]).parse
+      described_class.new(%w[a b]).parse
     }.to raise_exception(GovukSeedCrawler::CLIException, "too many arguments provided")
   end
 
-  it "should nest the help message in with any CLIExceptions we raise" do
+  it "nests the help message in with any CLIExceptions we raise" do
     expect {
-      GovukSeedCrawler::CLIParser.new(["a", "b"]).parse
+      described_class.new(%w[a b]).parse
     }.to raise_exception(GovukSeedCrawler::CLIException) { |e|
       expect(e.help).to include("Usage: ")
     }
@@ -31,48 +29,36 @@ describe GovukSeedCrawler::CLIParser do
   describe "catching STDOUT" do
     it "shows the help banner when provided -h" do
       # Get a valid options response as help closes early with SystemExit.
-      options = GovukSeedCrawler::CLIParser.new(["http://www.foo.com/"]).options
-
-      temp_stdout do |caught_stdout|
-        expect {
-          _, _ = GovukSeedCrawler::CLIParser.new(["-h"]).parse
-        }.to raise_exception(SystemExit) { |e|
-          expect(e.status).to eq(0)
-        }
+      options = described_class.new(["http://www.foo.com/"]).options
 
-        expect(caught_stdout.strip).to eq(options.help)
-      end
+      expect { described_class.new(["-h"]).parse }
+        .to output("#{options.help}\n").to_stdout
+        .and raise_exception(SystemExit) { |e| expect(e.status).to eq(0) }
     end
 
-    it "should show the version number and exit" do
-      temp_stdout do |caught_stdout|
-        expect {
-          _, _ = GovukSeedCrawler::CLIParser.new(["--version"]).parse
-        }.to raise_exception(SystemExit) { |e|
-          expect(e.status).to eq(0)
-        }
-
-        expect(caught_stdout.strip).to eq("Version: #{GovukSeedCrawler::VERSION}")
-      end
+    it "shows the version number and exit" do
+      expect { described_class.new(["--version"]).parse }
+        .to output("Version: #{GovukSeedCrawler::VERSION}\n").to_stdout
+        .and raise_exception(SystemExit) { |e| expect(e.status).to eq(0) }
     end
   end
 
   describe "passing in valid arguments" do
-    let(:arguments) {
+    let(:arguments) do
       [
-       "https://www.override.com/",
-       "--host rabbitmq.some.custom.vhost",
-       "--port 4567",
-       "--username foo",
-       "--password bar",
-       "--exchange some_custom_exchange",
-       "--topic some_custom_topic",
-       "--vhost a_vhost",
-       "--verbose"
+        "https://www.override.com/",
+        "--host rabbitmq.some.custom.vhost",
+        "--port 4567",
+        "--username foo",
+        "--password bar",
+        "--exchange some_custom_exchange",
+        "--topic some_custom_topic",
+        "--vhost a_vhost",
+        "--verbose",
       ].join(" ").split(" ")
-    }
+    end
 
-    it "should override all of the default arguments that we're providing" do
+    it "overrides all of the default arguments that we're providing" do
       overriden = {
         host: "rabbitmq.some.custom.vhost",
         port: "4567",
@@ -84,14 +70,14 @@ describe GovukSeedCrawler::CLIParser do
         quiet: false,
         verbose: true,
         version: nil,
-        vhost: "a_vhost"
+        vhost: "a_vhost",
       }
 
-      expect(GovukSeedCrawler::CLIParser.new(arguments).parse.first).to eq(overriden)
+      expect(described_class.new(arguments).parse.first).to eq(overriden)
     end
 
-    it "should set the --quiet value" do
-      options, _ = GovukSeedCrawler::CLIParser.new(["foo.com", "--quiet"]).parse
+    it "sets the --quiet value" do
+      options, = described_class.new(["foo.com", "--quiet"]).parse
       expect(options).to eq(GovukSeedCrawler::CLIParser::DEFAULTS.merge(quiet: true))
     end
 
@@ -107,14 +93,14 @@ describe GovukSeedCrawler::CLIParser do
       it "sets the password if set using an environment variable" do
         set_amqp_pass("foobar")
 
-        expect(GovukSeedCrawler::CLIParser.new(["http://www.example.com"]).parse.first)
+        expect(described_class.new(["http://www.example.com"]).parse.first)
           .to include(password: "foobar")
       end
 
       it "picks the environment variable over the parameter if both are set" do
         set_amqp_pass("bar")
 
-        expect(GovukSeedCrawler::CLIParser.new(["http://www.example.com", "--password", "foo"]).parse.first)
+        expect(described_class.new(["http://www.example.com", "--password", "foo"]).parse.first)
           .to include(password: "bar")
       end
     end

data/spec/govuk_seed_crawler/cli_runner_spec.rb

@@ -1,45 +1,25 @@
-require 'spec_helper'
-
 describe GovukSeedCrawler::CLIRunner do
   describe "printing the version" do
-    it "should not try to connect to an AMQP server" do
+    it "does not try to connect to an AMQP server" do
       expect(Bunny).not_to receive(:new)
 
-      temp_stdout do |caught_stdout|
-        expect {
-          GovukSeedCrawler::CLIRunner.new(["--version"]).run
-        }.to raise_exception(SystemExit) { |exit|
-          expect(exit.status).to eq(0)
-        }
-
-        expect(caught_stdout.strip).to eq("Version: #{GovukSeedCrawler::VERSION}")
-      end
+      expect { described_class.new(["--version"]).run }
+        .to output("Version: #{GovukSeedCrawler::VERSION}\n").to_stdout
+        .and raise_exception(SystemExit) { |e| expect(e.status).to eq(0) }
     end
   end
 
   describe "catching any CLIException objects and exiting with a status 1" do
     it "prints to STDOUT for too many arguments" do
-      temp_stdout do |caught_stdout|
-        expect {
-          GovukSeedCrawler::CLIRunner.new(["a", "b"])
-        }.to raise_exception(SystemExit) { |exit|
-          expect(exit.status).to eq(2)
-        }
-
-        expect(caught_stdout.strip).to include("too many arguments provided")
-      end
+      expect { described_class.new(%w[a b]).run }
+        .to output(/\Atoo many arguments provided/).to_stdout
+        .and raise_exception(SystemExit) { |e| expect(e.status).to eq(2) }
     end
 
     it "prints to STDOUT when site_root not set" do
-      temp_stdout do |caught_stdout|
-        expect {
-          GovukSeedCrawler::CLIRunner.new(["--verbose"])
-        }.to raise_exception(SystemExit) { |exit|
-          expect(exit.status).to eq(2)
-        }
-
-        expect(caught_stdout.strip).to include("site_root must be provided")
-      end
+      expect { described_class.new(["--verbose"]).run }
+        .to output(/\Asite_root must be provided/).to_stdout
+        .and raise_exception(SystemExit) { |e| expect(e.status).to eq(2) }
     end
   end
 
@@ -49,26 +29,26 @@ describe GovukSeedCrawler::CLIRunner do
     end
 
     it "defaults to INFO" do
-      GovukSeedCrawler::CLIRunner.new(["http://www.example.com"])
+      described_class.new(["http://www.example.com"])
       expect(GovukSeedCrawler.logger.level).to eq(Logger::INFO)
     end
 
     it "sets to ERROR for quite" do
-      GovukSeedCrawler::CLIRunner.new(["http://www.example.com", "--quiet"])
+      described_class.new(["http://www.example.com", "--quiet"])
       expect(GovukSeedCrawler.logger.level).to eq(Logger::ERROR)
     end
 
     it "sets to DEBUG for verbose" do
-      GovukSeedCrawler::CLIRunner.new(["http://www.example.com", "--verbose"])
+      described_class.new(["http://www.example.com", "--verbose"])
       expect(GovukSeedCrawler.logger.level).to eq(Logger::DEBUG)
     end
   end
 
   describe "#run" do
     it "passes all options through to seed" do
-      expect(GovukSeedCrawler::Seeder).to receive(:seed).
-        with("http://www.example.com", GovukSeedCrawler::CLIParser::DEFAULTS).once
-      GovukSeedCrawler::CLIRunner.new(["http://www.example.com"]).run
+      expect(GovukSeedCrawler::Seeder).to receive(:seed)
+        .with("http://www.example.com", GovukSeedCrawler::CLIParser::DEFAULTS).once
+      described_class.new(["http://www.example.com"]).run
     end
   end
 end

data/spec/govuk_seed_crawler/indexer_spec.rb

@@ -1,21 +1,18 @@
-require 'spec_helper'
-
 describe GovukSeedCrawler::Indexer do
-  subject { GovukSeedCrawler::Indexer.new('https://example.com') }
-
-  context "under normal usage" do
-    let(:mock_parser) do
-      double(:mock_parser, :to_a => [])
-    end
+  let(:mock_parser) { instance_double(SitemapParser, to_a: []) }
 
-    it "responds to Indexer#urls" do
-      allow(SitemapParser).to receive(:new).and_return(mock_parser)
-      expect(subject).to respond_to(:urls)
-    end
+  it "responds to Indexer#urls" do
+    allow(SitemapParser).to receive(:new).and_return(mock_parser)
+    instance = nil
+    expect { instance = described_class.new("https://example.com") }
+      .to output.to_stdout
+    expect(instance).to respond_to(:urls)
+  end
 
-    it "calls SitemapParser with the sitemap file" do
-      expect(SitemapParser).to receive(:new).with('https://example.com/sitemap.xml', {:recurse => true}).and_return(mock_parser)
-      subject
-    end
+  it "calls SitemapParser with the sitemap file" do
+    allow(SitemapParser).to receive(:new).with("https://example.com/sitemap.xml", { recurse: true }).and_return(mock_parser)
+    expect { described_class.new("https://example.com") }
+      .to output.to_stdout
+    expect(SitemapParser).to have_received(:new)
   end
 end

data/spec/govuk_seed_crawler/seeder_spec.rb

@@ -1,29 +1,27 @@
-require 'spec_helper'
-
 describe GovukSeedCrawler::Seeder do
   let(:exchange) { "seeder_test_exchange" }
   let(:topic) { "#" }
   let(:root_url) { "https://www.example.com" }
 
-  let(:options) {{
-    :exchange => exchange,
-    :topic => topic,
-  }}
+  let(:options) do
+    {
+      exchange: exchange,
+      topic: topic,
+    }
+  end
 
-  let(:mock_get_urls) { double(:mock_get_urls, :urls => true) }
-  let(:mock_amqp_client) { double(:mock_amqp_client, :close => true) }
+  let(:mock_get_urls) { instance_double(GovukSeedCrawler::Indexer, urls: true) }
+  let(:mock_amqp_client) { instance_double(GovukSeedCrawler::AmqpClient, close: true) }
 
   let(:urls) do
     [
-     "https://example.com/foo",
-     "https://example.com/bar",
-     "https://example.com/baz",
+      "https://example.com/foo",
+      "https://example.com/bar",
+      "https://example.com/baz",
     ]
   end
 
-  subject { GovukSeedCrawler::Seeder::seed(root_url, options) }
-
-  before(:each) do
+  before do
     allow(GovukSeedCrawler::Indexer).to receive(:new)
       .with(root_url)
       .and_return(mock_get_urls)
@@ -32,20 +30,20 @@ describe GovukSeedCrawler::Seeder do
       .with(options).and_return(mock_amqp_client)
   end
 
-  context "under normal usage" do
-    it "publishes urls to the queue" do
-      urls.each do |url|
-        expect(mock_amqp_client).to receive(:publish)
-          .with(exchange, topic, url)
-      end
-
-      subject
+  it "publishes urls to the queue" do
+    urls.each do |url|
+      expect(mock_amqp_client).to receive(:publish)
+        .with(exchange, topic, url)
     end
 
-    it "closes the connection when done" do
-      allow(mock_amqp_client).to receive(:publish)
-      expect(mock_amqp_client).to receive(:close)
-      subject
-    end
+    expect { described_class.seed(root_url, options) }
+      .to output.to_stdout
+  end
+
+  it "closes the connection when done" do
+    allow(mock_amqp_client).to receive(:publish)
+    expect(mock_amqp_client).to receive(:close)
+    expect { described_class.seed(root_url, options) }
+      .to output.to_stdout
   end
 end

data/spec/integration/govuk_seed_crawler_spec.rb

@@ -1,9 +1,8 @@
-require 'json'
-require 'spec_helper'
+require "json"
 
 describe GovukSeedCrawler do
   def stub_sitemap
-    sitemap = %{<?xml version="1.0" encoding="UTF-8"?>
+    sitemap = %(<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   <url>
     <loc>https://www.gov.uk/</loc>
@@ -15,45 +14,46 @@ describe GovukSeedCrawler do
     <loc>https://www.gov.uk/help</loc>
   </url>
 </urlset>
-    }
+    )
 
-    stub_request(:get, "https://www.gov.uk/sitemap.xml").
-         to_return(:status => 200, :body => sitemap, :headers => {})
+    stub_request(:get, "https://www.gov.uk/sitemap.xml")
+         .to_return(status: 200, body: sitemap, headers: {})
   end
 
+  subject { GovukSeedCrawler::Seeder.seed(site_root, options) }
+
   let(:vhost) { "/" }
   let(:exchange_name) { "govuk_seed_crawler_integration_exchange" }
   let(:queue_name) { "govuk_seed_crawler_integration_queue" }
   let(:topic) { "#" }
   let(:site_root) { "https://www.gov.uk" }
-  let(:options) {{
-      :host => ENV.fetch("AMQP_HOST", "localhost"),
-      :user => ENV.fetch("AMQP_USER", "govuk_seed_crawler"),
-      :pass => ENV.fetch("AMQP_PASS", "govuk_seed_crawler"),
-      :exchange => exchange_name,
-      :topic => topic
-  }}
-  let(:rabbitmq_client) { GovukSeedCrawler::AmqpClient.new(options) }
-
-  subject { GovukSeedCrawler::Seeder::seed(site_root, options) }
-
-  before(:each) do
-    @exchange = rabbitmq_client.channel.topic(exchange_name, :durable => true)
-    @queue = rabbitmq_client.channel.queue(queue_name)
-    @queue.bind(@exchange, :routing_key => topic)
+  let(:options) do
+    {
+      host: ENV.fetch("AMQP_HOST", "localhost"),
+      user: ENV.fetch("AMQP_USER", "govuk_seed_crawler"),
+      pass: ENV.fetch("AMQP_PASS", "govuk_seed_crawler"),
+      exchange: exchange_name,
+      topic: topic,
+    }
   end
-
-  after(:each) do
-    @queue.unbind(@exchange)
-    @queue.delete
-    @exchange.delete
+  let(:rabbitmq_client) { GovukSeedCrawler::AmqpClient.new(options) }
+  let(:exchange) { rabbitmq_client.channel.topic(exchange_name, durable: true) }
+  let(:queue) { rabbitmq_client.channel.queue(queue_name) }
+
+  around do |example|
+    queue.bind(exchange, routing_key: topic)
+    example.run
+    queue.unbind(exchange)
+    queue.delete
+    exchange.delete
     rabbitmq_client.close
   end
 
   it "publishes URLs it finds to an AMQP topic exchange" do
     stub_sitemap
-    subject
+    expect { GovukSeedCrawler::Seeder.seed(site_root, options) }
+      .to output.to_stdout
 
-    expect(@queue.message_count).to be(3)
+    expect(queue.message_count).to be(3)
   end
 end

data/spec/spec_helper.rb

@@ -1,5 +1,7 @@
-require 'govuk_seed_crawler'
-require 'webmock/rspec'
+require "govuk_seed_crawler"
+require "webmock/rspec"
+
+WebMock.disable_net_connect!
 
 RSpec.configure do |config|
   config.order = :random
@@ -27,13 +29,9 @@ RSpec.configure do |config|
     # a real object. This is generally recommended.
     mocks.verify_partial_doubles = true
   end
-end
 
-WebMock.disable_net_connect!
-
-def temp_stdout
-  $stdout = StringIO.new
-  yield $stdout.string
-ensure
-  $stdout = STDOUT
+  config.before do
+    # reset logger before each invocation so we can catch stdout
+    GovukSeedCrawler.logger = nil
+  end
 end

metadata

@@ -1,208 +1,207 @@
 --- !ruby/object:Gem::Specification
 name: govuk_seed_crawler
 version: !ruby/object:Gem::Version
-  version: 2.1.0
-  prerelease: 
+  version: 3.1.0
 platform: ruby
 authors:
 - GOV.UK developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-23 00:00:00.000000000 Z
+date: 2022-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.3'
-- !ruby/object:Gem::Dependency
-  name: crack
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.4.4
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - '='
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.4.4
+        version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: ffi
+  name: crack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.10.0
+        version: 0.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.10.0
+        version: 0.4.5
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: '1.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: '1.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.14'
 - !ruby/object:Gem::Dependency
   name: public_suffix
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.6
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.6
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.1.0
 - !ruby/object:Gem::Dependency
   name: sitemap-parser
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.6.0
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
   name: rspec-mocks
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-govuk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.8.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.18.0
+        version: '3.18'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.18.0
+        version: '3.18'
 description: 
 email:
 - govuk-dev@digital.cabinet-office.gov.uk
@@ -211,12 +210,15 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .ruby-version
+- ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- CHANGELOG.md
 - Gemfile
-- Jenkinsfile
-- LICENSE.txt
+- LICENCE
 - README.md
 - Rakefile
 - bin/seed-crawler
@@ -238,27 +240,25 @@ files:
 homepage: https://github.com/alphagov/govuk_seed_crawler
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.23.2
+rubygems_version: 3.3.24
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Retrieves a list of URLs to seed the crawler by publishing them to a RabbitMQ
   exchange.
 test_files:

data/Jenkinsfile

@@ -1,49 +0,0 @@
-#!/usr/bin/env groovy
-
-library("govuk")
-
-node {
-  try {
-    // This doesn't use the buildProject as this project doesn't conform to
-    // required norms (e.g. running in Ruby 1.9, non-standard tests).
-
-    repoName = JOB_NAME.split('/')[0]
-
-    stage("Checkout") {
-      govuk.checkoutFromGitHubWithSSH(repoName)
-    }
-
-    stage("Clean up workspace") {
-      govuk.cleanupGit()
-    }
-
-    stage('Configure environment') {
-      govuk.setEnvar('RBENV_VERSION', '1.9.3-p550')
-    }
-
-    stage('Bundle install') {
-      govuk.bundleGem()
-    }
-
-    stage('Spec tests') {
-      govuk.runRakeTask('spec')
-    }
-
-    stage('Integration tests') {
-      govuk.runRakeTask('integration')
-    }
-
-    if (env.BRANCH_NAME == 'master') {
-      stage('Publish Gem to Rubygems') {
-        govuk.publishGem(repoName, repoName, 'master')
-      }
-    }
-  } catch (e) {
-    currentBuild.result = "FAILED"
-    step([$class: 'Mailer',
-          notifyEveryUnstableBuild: true,
-          recipients: 'govuk-ci-notifications@digital.cabinet-office.gov.uk',
-          sendToIndividuals: true])
-    throw e
-  }
-}