govuk_seed_crawler 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a37f428070681bc4ca2466497df0b69f45fed94a
+  data.tar.gz: 55bdafe5ade9251f6f630eeb490e481a9796fc4f
+SHA512:
+  metadata.gz: 631f38d96a7d1ea301b38e761d5c55debbb9ee0c99a8e2a88ef1bb965b12637eceaaba70dfbd13941e4b96c088781fc6da3f724e283bfe8afc6d3dd8f0732321
+  data.tar.gz: 2f8b41afecdaba199b32925b13804c6dce588535b611a01383753dac2a22a655e728a9b77cf529845e0b6df1738bdca716c0eac5a5a4032e70e61d7d94c9cd82

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+2.1.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in govuk_seed_crawler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+(c) 2014 Crown copyright
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# GOV.UK: Seed the Crawler
+
+Retrieves a list of URLs to seed the [crawler](https://github.com/alphagov/govuk_crawler_worker) by publishing them to a RabbitMQ exchange.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'govuk_seed_crawler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install govuk_seed_crawler
+
+## Usage
+
+To run with the RabbitMQ connection defaults:
+
+```bash
+bundle exec seed-crawler https://www.gov.uk/
+```
+
+Run with `--help` to see a list of options:
+
+```bash
+bundle exec seed-crawler --help
+```
+
+## Contributing
+
+1. Fork it ( http://github.com/{my-github-username}/govuk_seed_crawler/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,18 @@
+require 'gem_publisher'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.pattern = FileList['spec/govuk_seed_crawler/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:integration) do |task|
+  task.pattern = FileList['spec/integration/**/*_spec.rb']
+end
+
+task :default => :spec
+
+desc "Publish gem to RubyGems"
+task :publish_gem do |t|
+  gem = GemPublisher.publish_if_updated("govuk_seed_crawler.gemspec")
+  puts "Published #{gem}" if gem
+end

data/bin/seed-crawler

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'govuk_seed_crawler'
+
+GovukSeedCrawler::CLIRunner.new(ARGV).run

data/govuk_seed_crawler.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'govuk_seed_crawler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "govuk_seed_crawler"
+  spec.version       = GovukSeedCrawler::VERSION
+  spec.authors       = ["Matt Bostock"]
+  spec.email         = ["matt.bostock@digital.cabinet-office.gov.uk"]
+  spec.summary       = %q{Retrieves a list of URLs to seed the crawler by publishing them to a RabbitMQ exchange.}
+  spec.homepage      = "https://github.gds/gds/govuk_seed_crawler"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "bunny", "~> 1.3"
+  spec.add_runtime_dependency "govuk_mirrorer", "~> 1.3.1"
+  spec.add_runtime_dependency "slop", "~> 3.6.0"
+
+  spec.add_development_dependency "gem_publisher", "~> 1.3"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rspec-mocks", "~> 3.0"
+  spec.add_development_dependency "webmock", "~> 1.18.0"
+end

data/jenkins-branches.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -e
+
+[ -x .venv/bin/pip ] || virtualenv .venv
+. .venv/bin/activate
+
+pip install -q ghtools
+
+REPO="gds:gds/govuk_seed_crawler"
+gh-status "$REPO" "$GIT_COMMIT" pending -d "\"Build #${BUILD_NUMBER} is running on Jenkins\"" -u "$BUILD_URL" >/dev/null
+
+if ./jenkins-tests.sh; then
+  gh-status "$REPO" "$GIT_COMMIT" success -d "\"Build #${BUILD_NUMBER} succeeded on Jenkins\"" -u "$BUILD_URL" >/dev/null
+  exit 0
+else
+  gh-status "$REPO" "$GIT_COMMIT" failure -d "\"Build #${BUILD_NUMBER} failed on Jenkins\"" -u "$BUILD_URL" >/dev/null
+  exit 1
+fi

data/jenkins-tests.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -e
+rm -f Gemfile.lock
+bundle install --path "${HOME}/bundles/${JOB_NAME}"
+bundle exec rake
+bundle exec rake integration

data/jenkins.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -e
+
+./jenkins-tests.sh
+bundle exec rake publish_gem

data/lib/govuk_seed_crawler.rb

@@ -0,0 +1,17 @@
+require 'govuk_seed_crawler/amqp_client'
+require 'govuk_seed_crawler/cli_parser'
+require 'govuk_seed_crawler/cli_runner'
+require 'govuk_seed_crawler/indexer'
+require 'govuk_seed_crawler/seeder'
+require 'govuk_seed_crawler/version'
+
+module GovukSeedCrawler
+  def self.logger
+    unless @logger
+      @logger = Logger.new(STDOUT)
+      @logger.level = Logger::INFO
+    end
+
+    @logger
+  end
+end

data/lib/govuk_seed_crawler/amqp_client.rb

@@ -0,0 +1,28 @@
+require 'bunny'
+
+module GovukSeedCrawler
+  class AmqpClient
+    attr_reader :channel
+
+    def initialize(connection_options = {})
+      @conn = Bunny.new(connection_options)
+      @conn.start
+      @channel = @conn.create_channel
+    end
+
+    def close
+      @conn.close
+    end
+
+    def publish(exchange, topic, body)
+      raise "Exchange cannot be nil" if exchange.nil?
+      raise "Topic cannot be nil" if topic.nil?
+      raise "Message body cannot be nil" if body.nil?
+
+      GovukSeedCrawler.logger.debug("Publishing '#{body}' to topic '#{topic}'")
+
+      @channel.topic(exchange, :durable => true)
+        .publish(body, :routing_key => topic)
+    end
+  end
+end

data/lib/govuk_seed_crawler/cli_parser.rb

@@ -0,0 +1,81 @@
+require 'slop'
+
+module GovukSeedCrawler
+  class CLIException < StandardError
+    attr_reader :help
+
+    def initialize(message, help)
+      super(message)
+      @help = help
+    end
+  end
+
+  class CLIParser
+    DEFAULTS = {
+      :exchange => "govuk_crawler_exchange",
+      :help => nil,
+      :host => "localhost",
+      :password => "guest",
+      :port => "5672",
+      :quiet => false,
+      :topic => "#",
+      :username => "guest",
+      :verbose => false,
+      :version => nil,
+      :vhost => "/"
+    }.freeze
+
+    ENV_AMQP_PASS_KEY = "GOVUK_CRAWLER_AMQP_PASS".freeze
+
+    def initialize(argv_array)
+      @argv_array = argv_array
+    end
+
+    def options
+      Slop.parse!(@argv_array, :help => true) do
+        banner <<-EOS
+Usage: #{$PROGRAM_NAME} site_root [options]
+
+Seeds an AMQP topic exchange with messages, each containing a URL, for the GOV.UK Crawler Worker
+to consume:
+
+https://github.com/alphagov/govuk_crawler_worker
+
+The AMQP password can also be set as an environment variable and will be read from
+`#{ENV_AMQP_PASS_KEY}`. If both the environment variable and command-line option for password
+are set, the environment variable will take higher precedent.
+        EOS
+
+        on :version, "Display version and exit" do
+          puts "Version: #{GovukSeedCrawler::VERSION}"
+          exit 0
+        end
+
+        on :host=, "AMQP host to publish to", default: DEFAULTS[:host]
+        on :port=, "AMQP port", default: DEFAULTS[:port]
+        on :username=, "AMQP username", default: DEFAULTS[:username]
+        on :password=, "AMQP password", default: DEFAULTS[:password]
+        on :exchange=, "AMQP exchange", default: DEFAULTS[:exchange]
+        on :topic=, "AMQP topic", default: DEFAULTS[:topic]
+        on :vhost=, "AMQP vhost", default: DEFAULTS[:vhost]
+
+        on :quiet, "Quiet output", default: DEFAULTS[:quiet]
+        on :verbose, "Verbose output", default: DEFAULTS[:verbose]
+      end
+    end
+
+    def parse
+      opts = options
+
+      if opts[:version].nil?
+        raise CLIException.new("too many arguments provided", opts.help) if @argv_array.size > 1
+        raise CLIException.new("site_root must be provided", opts.help) if @argv_array.size != 1
+      end
+
+      options_hash = opts.to_hash
+      options_hash[:password] = ENV[ENV_AMQP_PASS_KEY] unless ENV[ENV_AMQP_PASS_KEY].nil?
+
+      return options_hash, @argv_array.first
+    end
+  end
+end

data/lib/govuk_seed_crawler/cli_runner.rb

@@ -0,0 +1,29 @@
+module GovukSeedCrawler
+  class CLIRunner
+    def initialize(argv_array)
+      begin
+        @options, @site_root = CLIParser.new(argv_array).parse
+      rescue CLIException => e
+        puts e.message
+        puts e.help
+        exit 2
+      end
+
+      set_logging_level(@options)
+    end
+
+    def run
+      Seeder::seed(@site_root, @options)
+    end
+
+    private
+
+    def set_logging_level(cli_options)
+      if cli_options[:verbose]
+        GovukSeedCrawler.logger.level = Logger::DEBUG
+      elsif cli_options[:quiet]
+        GovukSeedCrawler.logger.level = Logger::ERROR
+      end
+    end
+  end
+end

data/lib/govuk_seed_crawler/indexer.rb

@@ -0,0 +1,18 @@
+require 'govuk_mirrorer/indexer'
+require 'govuk_mirrorer/statsd'
+
+module GovukSeedCrawler
+  class Indexer
+    attr_reader :urls
+
+    def initialize(site_root)
+      raise "No site_root defined" unless site_root
+
+      GovukSeedCrawler.logger.info("Retrieving list of URLs for #{site_root}")
+      indexer = GovukMirrorer::Indexer.new(site_root)
+      @urls = indexer.all_start_urls
+
+      GovukSeedCrawler.logger.info("Found #{@urls.count} URLs")
+    end
+  end
+end

data/lib/govuk_seed_crawler/seeder.rb

@@ -0,0 +1,16 @@
+module GovukSeedCrawler
+  class Seeder
+    def self.seed(site_root, options = {})
+      amqp_client = AmqpClient.new(options)
+      urls = Indexer.new(site_root).urls
+
+      urls.each do |url|
+        amqp_client.publish(options[:exchange], options[:topic], url)
+      end
+
+      GovukSeedCrawler.logger.info("Published #{urls.count} URLs to topic '#{options[:topic]}'")
+
+      amqp_client.close
+    end
+  end
+end

data/lib/govuk_seed_crawler/version.rb

@@ -0,0 +1,3 @@
+module GovukSeedCrawler
+  VERSION = "1.0.0"
+end

data/spec/govuk_seed_crawler/amqp_client_spec.rb

@@ -0,0 +1,55 @@
+require 'spec_helper'
+
+describe GovukSeedCrawler::AmqpClient do
+  let(:exchange) { "govuk_seed_crawler_spec_exchange" }
+  let(:options) {{
+    :host => ENV.fetch("AMQP_HOST", "localhost"),
+    :user => ENV.fetch("AMQP_USER", "govuk_seed_crawler"),
+    :pass => ENV.fetch("AMQP_PASS", "govuk_seed_crawler"),
+  }}
+  subject { GovukSeedCrawler::AmqpClient.new(options) }
+
+  it "responds to #channel" do
+    expect(subject).to respond_to(:channel)
+  end
+
+  it "responds to #close" do
+    expect(subject).to respond_to(:close)
+  end
+
+  it "closes the connection to the AMQP server" do
+    mock_bunny = double(:mock_bunny,
+      :start => true, :create_channel => true, :close => true)
+    allow(Bunny).to receive(:new).and_return(mock_bunny)
+    expect(mock_bunny).to receive(:close).once
+
+    subject.close
+  end
+
+  context "#publish" do
+    context "error handling" do
+      it "raises an exception if exchange is nil" do
+        expect {
+          subject.publish(nil, "#", "some body")
+        }.to raise_exception(RuntimeError, "Exchange cannot be nil")
+      end
+
+      it "raises an exception if topic is nil" do
+        expect {
+          subject.publish(exchange, nil, "some body")
+        }.to raise_exception(RuntimeError, "Topic cannot be nil")
+      end
+
+      it "raises an exception if body is nil" do
+        expect {
+          subject.publish(exchange, "#", nil)
+        }.to raise_exception(RuntimeError, "Message body cannot be nil")
+      end
+    end
+
+    it "allows publishing against an exchange" do
+      expect(subject.publish(exchange, "#", "some body"))
+        .to_not be_nil
+    end
+  end
+end

data/spec/govuk_seed_crawler/cli_parser_spec.rb

@@ -0,0 +1,122 @@
+require 'spec_helper'
+
+describe GovukSeedCrawler::CLIParser do
+  it "requires the site_root to be provided" do
+    expect {
+      GovukSeedCrawler::CLIParser.new([]).parse
+    }.to raise_exception(GovukSeedCrawler::CLIException, "site_root must be provided")
+  end
+
+  it "provides the defaults when just given the site_root" do
+    options, site_root = GovukSeedCrawler::CLIParser.new(["https://www.example.com"]).parse
+
+    expect(options).to eq(GovukSeedCrawler::CLIParser::DEFAULTS)
+    expect(site_root).to eq("https://www.example.com")
+  end
+
+  it "should tell us when we've given too many arguments" do
+    expect {
+      GovukSeedCrawler::CLIParser.new(["a", "b"]).parse
+    }.to raise_exception(GovukSeedCrawler::CLIException, "too many arguments provided")
+  end
+
+  it "should nest the help message in with any CLIExceptions we raise" do
+    expect {
+      GovukSeedCrawler::CLIParser.new(["a", "b"]).parse
+    }.to raise_exception(GovukSeedCrawler::CLIException) { |e|
+      expect(e.help).to include("Usage: ")
+    }
+  end
+
+  describe "catching STDOUT" do
+    it "shows the help banner when provided -h" do
+      # Get a valid options response as help closes early with SystemExit.
+      options = GovukSeedCrawler::CLIParser.new(["http://www.foo.com/"]).options
+
+      temp_stdout do |caught_stdout|
+        expect {
+          _, _ = GovukSeedCrawler::CLIParser.new(["-h"]).parse
+        }.to raise_exception(SystemExit) { |e|
+          expect(e.status).to eq(0)
+        }
+
+        expect(caught_stdout.strip).to eq(options.help)
+      end
+    end
+
+    it "should show the version number and exit" do
+      temp_stdout do |caught_stdout|
+        expect {
+          _, _ = GovukSeedCrawler::CLIParser.new(["--version"]).parse
+        }.to raise_exception(SystemExit) { |e|
+          expect(e.status).to eq(0)
+        }
+
+        expect(caught_stdout.strip).to eq("Version: #{GovukSeedCrawler::VERSION}")
+      end
+    end
+  end
+
+  describe "passing in valid arguments" do
+    let(:arguments) {
+      [
+       "https://www.override.com/",
+       "--host rabbitmq.some.custom.vhost",
+       "--port 4567",
+       "--username foo",
+       "--password bar",
+       "--exchange some_custom_exchange",
+       "--topic some_custom_topic",
+       "--vhost a_vhost",
+       "--verbose"
+      ].join(" ").split(" ")
+    }
+
+    it "should override all of the default arguments that we're providing" do
+      overriden = {
+        host: "rabbitmq.some.custom.vhost",
+        port: "4567",
+        username: "foo",
+        password: "bar",
+        exchange: "some_custom_exchange",
+        topic: "some_custom_topic",
+        help: nil,
+        quiet: false,
+        verbose: true,
+        version: nil,
+        vhost: "a_vhost"
+      }
+
+      expect(GovukSeedCrawler::CLIParser.new(arguments).parse.first).to eq(overriden)
+    end
+
+    it "should set the --quiet value" do
+      options, _ = GovukSeedCrawler::CLIParser.new(["foo.com", "--quiet"]).parse
+      expect(options).to eq(GovukSeedCrawler::CLIParser::DEFAULTS.merge(quiet: true))
+    end
+
+    describe "reading the AMQP password from an environment variable" do
+      def set_amqp_pass(password)
+        ENV[GovukSeedCrawler::CLIParser::ENV_AMQP_PASS_KEY] = password
+      end
+
+      after do
+        ENV[GovukSeedCrawler::CLIParser::ENV_AMQP_PASS_KEY] = nil
+      end
+
+      it "sets the password if set using an environment variable" do
+        set_amqp_pass("foobar")
+
+        expect(GovukSeedCrawler::CLIParser.new(["http://www.example.com"]).parse.first)
+          .to include(password: "foobar")
+      end
+
+      it "picks the environment variable over the parameter if both are set" do
+        set_amqp_pass("bar")
+
+        expect(GovukSeedCrawler::CLIParser.new(["http://www.example.com", "--password", "foo"]).parse.first)
+          .to include(password: "bar")
+      end
+    end
+  end
+end

data/spec/govuk_seed_crawler/cli_runner_spec.rb

@@ -0,0 +1,74 @@
+require 'spec_helper'
+
+describe GovukSeedCrawler::CLIRunner do
+  describe "printing the version" do
+    it "should not try to connect to an AMQP server" do
+      expect(Bunny).not_to receive(:new)
+
+      temp_stdout do |caught_stdout|
+        expect {
+          GovukSeedCrawler::CLIRunner.new(["--version"]).run
+        }.to raise_exception(SystemExit) { |exit|
+          expect(exit.status).to eq(0)
+        }
+
+        expect(caught_stdout.strip).to eq("Version: #{GovukSeedCrawler::VERSION}")
+      end
+    end
+  end
+
+  describe "catching any CLIException objects and exiting with a status 1" do
+    it "prints to STDOUT for too many arguments" do
+      temp_stdout do |caught_stdout|
+        expect {
+          GovukSeedCrawler::CLIRunner.new(["a", "b"])
+        }.to raise_exception(SystemExit) { |exit|
+          expect(exit.status).to eq(2)
+        }
+
+        expect(caught_stdout.strip).to include("too many arguments provided")
+      end
+    end
+
+    it "prints to STDOUT when site_root not set" do
+      temp_stdout do |caught_stdout|
+        expect {
+          GovukSeedCrawler::CLIRunner.new(["--verbose"])
+        }.to raise_exception(SystemExit) { |exit|
+          expect(exit.status).to eq(2)
+        }
+
+        expect(caught_stdout.strip).to include("site_root must be provided")
+      end
+    end
+  end
+
+  describe "setting the logging level" do
+    before do
+      GovukSeedCrawler.logger.level = Logger::INFO
+    end
+
+    it "defaults to INFO" do
+      GovukSeedCrawler::CLIRunner.new(["http://www.example.com"])
+      expect(GovukSeedCrawler.logger.level).to eq(Logger::INFO)
+    end
+
+    it "sets to ERROR for quite" do
+      GovukSeedCrawler::CLIRunner.new(["http://www.example.com", "--quiet"])
+      expect(GovukSeedCrawler.logger.level).to eq(Logger::ERROR)
+    end
+
+    it "sets to DEBUG for verbose" do
+      GovukSeedCrawler::CLIRunner.new(["http://www.example.com", "--verbose"])
+      expect(GovukSeedCrawler.logger.level).to eq(Logger::DEBUG)
+    end
+  end
+
+  describe "#run" do
+    it "passes all options through to seed" do
+      expect(GovukSeedCrawler::Seeder).to receive(:seed).
+        with("http://www.example.com", GovukSeedCrawler::CLIParser::DEFAULTS).once
+      GovukSeedCrawler::CLIRunner.new(["http://www.example.com"]).run
+    end
+  end
+end

data/spec/govuk_seed_crawler/indexer_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe GovukSeedCrawler::Indexer do
+  subject { GovukSeedCrawler::Indexer.new('https://example.com/') }
+
+  context "under normal usage" do
+    let(:mock_indexer) do
+      double(:mock_indexer, :all_start_urls => [])
+    end
+
+    it "responds to Indexer#urls" do
+      allow(GovukMirrorer::Indexer).to receive(:new).and_return(mock_indexer)
+      expect(subject).to respond_to(:urls)
+    end
+
+    it "calls GovukMirrorer::Indexer with the site root" do
+      expect(GovukMirrorer::Indexer).to receive(:new).with('https://example.com/').and_return(mock_indexer)
+      subject
+    end
+  end
+end

data/spec/govuk_seed_crawler/seeder_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+describe GovukSeedCrawler::Seeder do
+  let(:exchange) { "seeder_test_exchange" }
+  let(:topic) { "#" }
+  let(:root_url) { "https://www.example.com" }
+
+  let(:options) {{
+    :exchange => exchange,
+    :topic => topic,
+  }}
+
+  let(:mock_get_urls) { double(:mock_get_urls, :urls => true) }
+  let(:mock_amqp_client) { double(:mock_amqp_client, :close => true) }
+
+  let(:urls) do
+    [
+     "https://example.com/foo",
+     "https://example.com/bar",
+     "https://example.com/baz",
+    ]
+  end
+
+  subject { GovukSeedCrawler::Seeder::seed(root_url, options) }
+
+  before(:each) do
+    allow(GovukSeedCrawler::Indexer).to receive(:new)
+      .with(root_url)
+      .and_return(mock_get_urls)
+    allow(mock_get_urls).to receive(:urls).and_return(urls)
+    allow(GovukSeedCrawler::AmqpClient).to receive(:new)
+      .with(options).and_return(mock_amqp_client)
+  end
+
+  context "under normal usage" do
+    it "publishes urls to the queue" do
+      urls.each do |url|
+        expect(mock_amqp_client).to receive(:publish)
+          .with(exchange, topic, url)
+      end
+
+      subject
+    end
+
+    it "closes the connection when done" do
+      allow(mock_amqp_client).to receive(:publish)
+      expect(mock_amqp_client).to receive(:close)
+      subject
+    end
+  end
+end

data/spec/integration/govuk_seed_crawler_spec.rb

@@ -0,0 +1,66 @@
+require 'json'
+require 'spec_helper'
+
+describe GovukSeedCrawler do
+  def stub_api_artefacts(count)
+    item = {
+      "id" => "https://www.gov.uk/api/government%2Fnews%2Ffaster-review-of-support-for-renewable-electricity-to-provide-investor-certainty.json",
+      "web_url" => "https://www.gov.uk/government/news/faster-review-of-support-for-renewable-electricity-to-provide-investor-certainty",
+      "title" => "Faster review of support for Renewable electricity to provide investor certainty",
+      "format" => "announcement"
+    }
+    results = count.times.collect { item }
+    response = {
+      "_response_info" => {
+        "status" => "ok",
+        "links" => []
+      },
+      "total" => results.size,
+      "start_index" => 1,
+      "page_size" => 100,
+      "current_page" => 1,
+      "pages" => 1,
+      "results" => results
+    }
+
+    stub_request(:get, "https://www.gov.uk//api/artefacts.json").
+         to_return(:status => 200, :body => response.to_json, :headers => {})
+  end
+
+  let(:vhost) { "/" }
+  let(:exchange_name) { "govuk_seed_crawler_integration_exchange" }
+  let(:queue_name) { "govuk_seed_crawler_integration_queue" }
+  let(:topic) { "#" }
+  let(:site_root) { "https://www.gov.uk/" }
+  let(:options) {{
+      :host => ENV.fetch("AMQP_HOST", "localhost"),
+      :user => ENV.fetch("AMQP_USER", "govuk_seed_crawler"),
+      :pass => ENV.fetch("AMQP_PASS", "govuk_seed_crawler"),
+      :exchange => exchange_name,
+      :topic => topic
+  }}
+  let(:rabbitmq_client) { GovukSeedCrawler::AmqpClient.new(options) }
+
+  subject { GovukSeedCrawler::Seeder::seed(site_root, options) }
+
+  before(:each) do
+    @exchange = rabbitmq_client.channel.topic(exchange_name, :durable => true)
+    @queue = rabbitmq_client.channel.queue(queue_name)
+    @queue.bind(@exchange, :routing_key => topic)
+  end
+
+  after(:each) do
+    @queue.unbind(@exchange)
+    @queue.delete
+    @exchange.delete
+    rabbitmq_client.close
+  end
+
+  it "publishes URLs it finds to an AMQP topic exchange" do
+    stub_api_artefacts(10)
+    subject
+
+    # There's an extra 5 URLs from the Indexer class that are hard-coded.
+    expect(@queue.message_count).to be(15)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,39 @@
+require 'govuk_seed_crawler'
+require 'webmock/rspec'
+
+RSpec.configure do |config|
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+
+  config.expect_with :rspec do |expectations|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+    expectations.syntax = :expect
+  end
+
+  config.mock_with :rspec do |mocks|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+    mocks.syntax = :expect
+
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended.
+    mocks.verify_partial_doubles = true
+  end
+end
+
+WebMock.disable_net_connect!
+
+def temp_stdout
+  $stdout = StringIO.new
+  yield $stdout.string
+ensure
+  $stdout = STDOUT
+end

metadata

@@ -0,0 +1,205 @@
+--- !ruby/object:Gem::Specification
+name: govuk_seed_crawler
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Matt Bostock
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: govuk_mirrorer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+- !ruby/object:Gem::Dependency
+  name: gem_publisher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-mocks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.18.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.18.0
+description: 
+email:
+- matt.bostock@digital.cabinet-office.gov.uk
+executables:
+- seed-crawler
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/seed-crawler
+- govuk_seed_crawler.gemspec
+- jenkins-branches.sh
+- jenkins-tests.sh
+- jenkins.sh
+- lib/govuk_seed_crawler.rb
+- lib/govuk_seed_crawler/amqp_client.rb
+- lib/govuk_seed_crawler/cli_parser.rb
+- lib/govuk_seed_crawler/cli_runner.rb
+- lib/govuk_seed_crawler/indexer.rb
+- lib/govuk_seed_crawler/seeder.rb
+- lib/govuk_seed_crawler/version.rb
+- spec/govuk_seed_crawler/amqp_client_spec.rb
+- spec/govuk_seed_crawler/cli_parser_spec.rb
+- spec/govuk_seed_crawler/cli_runner_spec.rb
+- spec/govuk_seed_crawler/indexer_spec.rb
+- spec/govuk_seed_crawler/seeder_spec.rb
+- spec/integration/govuk_seed_crawler_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.gds/gds/govuk_seed_crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Retrieves a list of URLs to seed the crawler by publishing them to a RabbitMQ
+  exchange.
+test_files:
+- spec/govuk_seed_crawler/amqp_client_spec.rb
+- spec/govuk_seed_crawler/cli_parser_spec.rb
+- spec/govuk_seed_crawler/cli_runner_spec.rb
+- spec/govuk_seed_crawler/indexer_spec.rb
+- spec/govuk_seed_crawler/seeder_spec.rb
+- spec/integration/govuk_seed_crawler_spec.rb
+- spec/spec_helper.rb