govuk_seed_crawler 1.0.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a37f428070681bc4ca2466497df0b69f45fed94a
-  data.tar.gz: 55bdafe5ade9251f6f630eeb490e481a9796fc4f
+SHA256:
+  metadata.gz: 2ee27868f4df61a72b764044b8d04ad6f6b252a63cd1f365d1056a7d331a85d1
+  data.tar.gz: 514ee036b88544be5a7935dcc2103e07aa7fe82c2acc0841109d15e572a9ebf9
 SHA512:
-  metadata.gz: 631f38d96a7d1ea301b38e761d5c55debbb9ee0c99a8e2a88ef1bb965b12637eceaaba70dfbd13941e4b96c088781fc6da3f724e283bfe8afc6d3dd8f0732321
-  data.tar.gz: 2f8b41afecdaba199b32925b13804c6dce588535b611a01383753dac2a22a655e728a9b77cf529845e0b6df1738bdca716c0eac5a5a4032e70e61d7d94c9cd82
+  metadata.gz: 3d005f87f519187b619e1dbbeabcc441d0ccb0daf652db2d6b515f7abf2de49e9090f457e208afc5128d68db65bf618dcbf5bc74e54bb31c16bf6dbf405495c1
+  data.tar.gz: ec43d4205fd5714be7ab39669a9397371792802b3f50fa97d580c7327db018afd8409988b54c5876aba603456cd3019c454ce41736dad054e5f58d7cc79a80ed

data/.ruby-version

@@ -1 +1 @@
-2.1.2
+2.6.3

data/Jenkinsfile

@@ -0,0 +1,49 @@
+#!/usr/bin/env groovy
+
+library("govuk")
+
+node {
+  try {
+    // This doesn't use the buildProject as this project doesn't conform to
+    // required norms (e.g. running in Ruby 1.9, non-standard tests).
+
+    repoName = JOB_NAME.split('/')[0]
+
+    stage("Checkout") {
+      govuk.checkoutFromGitHubWithSSH(repoName)
+    }
+
+    stage("Clean up workspace") {
+      govuk.cleanupGit()
+    }
+
+    stage('Configure environment') {
+      govuk.setEnvar('RBENV_VERSION', '2.6.3')
+    }
+
+    stage('Bundle install') {
+      govuk.bundleGem()
+    }
+
+    stage('Spec tests') {
+      govuk.runRakeTask('spec')
+    }
+
+    stage('Integration tests') {
+      govuk.runRakeTask('integration')
+    }
+
+    if (env.BRANCH_NAME == 'master') {
+      stage('Publish Gem to Rubygems') {
+        govuk.publishGem(repoName, repoName, 'master')
+      }
+    }
+  } catch (e) {
+    currentBuild.result = "FAILED"
+    step([$class: 'Mailer',
+          notifyEveryUnstableBuild: true,
+          recipients: 'govuk-ci-notifications@digital.cabinet-office.gov.uk',
+          sendToIndividuals: true])
+    throw e
+  }
+}

data/README.md

@@ -1,6 +1,7 @@
 # GOV.UK: Seed the Crawler
 
-Retrieves a list of URLs to seed the [crawler](https://github.com/alphagov/govuk_crawler_worker) by publishing them to a RabbitMQ exchange.
+This gem retrieves a list of seed URLs from the GOV.UK sitemap and adds them to RabbitMQ
+so that the [crawler](https://github.com/alphagov/govuk_crawler_worker) can consume them.
 
 ## Installation
 

data/Rakefile

@@ -1,4 +1,3 @@
-require 'gem_publisher'
 require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec) do |task|
@@ -10,9 +9,3 @@ RSpec::Core::RakeTask.new(:integration) do |task|
 end
 
 task :default => :spec
-
-desc "Publish gem to RubyGems"
-task :publish_gem do |t|
-  gem = GemPublisher.publish_if_updated("govuk_seed_crawler.gemspec")
-  puts "Published #{gem}" if gem
-end

data/govuk_seed_crawler.gemspec

@@ -6,24 +6,30 @@ require 'govuk_seed_crawler/version'
 Gem::Specification.new do |spec|
   spec.name          = "govuk_seed_crawler"
   spec.version       = GovukSeedCrawler::VERSION
-  spec.authors       = ["Matt Bostock"]
-  spec.email         = ["matt.bostock@digital.cabinet-office.gov.uk"]
+  spec.authors       = ['GOV.UK developers']
+  spec.email         = ["govuk-dev@digital.cabinet-office.gov.uk"]
   spec.summary       = %q{Retrieves a list of URLs to seed the crawler by publishing them to a RabbitMQ exchange.}
-  spec.homepage      = "https://github.gds/gds/govuk_seed_crawler"
+  spec.homepage      = "https://github.com/alphagov/govuk_seed_crawler"
   spec.license       = "MIT"
 
+  spec.required_ruby_version = "~> 2.6"
+
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "bunny", "~> 1.3"
-  spec.add_runtime_dependency "govuk_mirrorer", "~> 1.3.1"
+  spec.add_runtime_dependency "crack", "0.4.4"
+  spec.add_runtime_dependency "nokogiri", "~> 1.6.0"
+  # Something, somewhere, sometimes requires public_suffix.
+  # public_suffix > 1.5 requires ruby > 2.
+  spec.add_runtime_dependency "public_suffix", "~> 1.4.6"
+  spec.add_runtime_dependency "sitemap-parser", "~> 0.3.0"
   spec.add_runtime_dependency "slop", "~> 3.6.0"
 
-  spec.add_development_dependency "gem_publisher", "~> 1.3"
   spec.add_development_dependency "pry"
-  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rake", "~> 0.9"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-mocks", "~> 3.0"
   spec.add_development_dependency "webmock", "~> 1.18.0"

data/lib/govuk_seed_crawler/indexer.rb

@@ -1,5 +1,4 @@
-require 'govuk_mirrorer/indexer'
-require 'govuk_mirrorer/statsd'
+require 'sitemap-parser'
 
 module GovukSeedCrawler
   class Indexer
@@ -9,8 +8,9 @@ module GovukSeedCrawler
       raise "No site_root defined" unless site_root
 
       GovukSeedCrawler.logger.info("Retrieving list of URLs for #{site_root}")
-      indexer = GovukMirrorer::Indexer.new(site_root)
-      @urls = indexer.all_start_urls
+
+      sitemap = SitemapParser.new("#{site_root}/sitemap.xml", {recurse: true})
+      @urls = sitemap.to_a
 
       GovukSeedCrawler.logger.info("Found #{@urls.count} URLs")
     end

data/lib/govuk_seed_crawler/version.rb

@@ -1,3 +1,3 @@
 module GovukSeedCrawler
-  VERSION = "1.0.0"
+  VERSION = "3.0.0"
 end

data/spec/govuk_seed_crawler/indexer_spec.rb

@@ -1,20 +1,20 @@
 require 'spec_helper'
 
 describe GovukSeedCrawler::Indexer do
-  subject { GovukSeedCrawler::Indexer.new('https://example.com/') }
+  subject { GovukSeedCrawler::Indexer.new('https://example.com') }
 
   context "under normal usage" do
-    let(:mock_indexer) do
-      double(:mock_indexer, :all_start_urls => [])
+    let(:mock_parser) do
+      double(:mock_parser, :to_a => [])
     end
 
     it "responds to Indexer#urls" do
-      allow(GovukMirrorer::Indexer).to receive(:new).and_return(mock_indexer)
+      allow(SitemapParser).to receive(:new).and_return(mock_parser)
       expect(subject).to respond_to(:urls)
     end
 
-    it "calls GovukMirrorer::Indexer with the site root" do
-      expect(GovukMirrorer::Indexer).to receive(:new).with('https://example.com/').and_return(mock_indexer)
+    it "calls SitemapParser with the sitemap file" do
+      expect(SitemapParser).to receive(:new).with('https://example.com/sitemap.xml', {:recurse => true}).and_return(mock_parser)
       subject
     end
   end

data/spec/integration/govuk_seed_crawler_spec.rb

@@ -2,36 +2,30 @@ require 'json'
 require 'spec_helper'
 
 describe GovukSeedCrawler do
-  def stub_api_artefacts(count)
-    item = {
-      "id" => "https://www.gov.uk/api/government%2Fnews%2Ffaster-review-of-support-for-renewable-electricity-to-provide-investor-certainty.json",
-      "web_url" => "https://www.gov.uk/government/news/faster-review-of-support-for-renewable-electricity-to-provide-investor-certainty",
-      "title" => "Faster review of support for Renewable electricity to provide investor certainty",
-      "format" => "announcement"
-    }
-    results = count.times.collect { item }
-    response = {
-      "_response_info" => {
-        "status" => "ok",
-        "links" => []
-      },
-      "total" => results.size,
-      "start_index" => 1,
-      "page_size" => 100,
-      "current_page" => 1,
-      "pages" => 1,
-      "results" => results
+  def stub_sitemap
+    sitemap = %{<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>https://www.gov.uk/</loc>
+  </url>
+  <url>
+    <loc>https://www.gov.uk/register-to-vote</loc>
+  </url>
+  <url>
+    <loc>https://www.gov.uk/help</loc>
+  </url>
+</urlset>
     }
 
-    stub_request(:get, "https://www.gov.uk//api/artefacts.json").
-         to_return(:status => 200, :body => response.to_json, :headers => {})
+    stub_request(:get, "https://www.gov.uk/sitemap.xml").
+         to_return(:status => 200, :body => sitemap, :headers => {})
   end
 
   let(:vhost) { "/" }
   let(:exchange_name) { "govuk_seed_crawler_integration_exchange" }
   let(:queue_name) { "govuk_seed_crawler_integration_queue" }
   let(:topic) { "#" }
-  let(:site_root) { "https://www.gov.uk/" }
+  let(:site_root) { "https://www.gov.uk" }
   let(:options) {{
       :host => ENV.fetch("AMQP_HOST", "localhost"),
       :user => ENV.fetch("AMQP_USER", "govuk_seed_crawler"),
@@ -57,10 +51,9 @@ describe GovukSeedCrawler do
   end
 
   it "publishes URLs it finds to an AMQP topic exchange" do
-    stub_api_artefacts(10)
+    stub_sitemap
     subject
 
-    # There's an extra 5 URLs from the Indexer class that are hard-coded.
-    expect(@queue.message_count).to be(15)
+    expect(@queue.message_count).to be(3)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: govuk_seed_crawler
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 3.0.0
 platform: ruby
 authors:
-- Matt Bostock
+- GOV.UK developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-28 00:00:00.000000000 Z
+date: 2021-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -25,47 +25,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: govuk_mirrorer
+  name: crack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.4.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.4.4
+- !ruby/object:Gem::Dependency
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: 1.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: 1.6.0
 - !ruby/object:Gem::Dependency
-  name: slop
+  name: public_suffix
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.6.0
+        version: 1.4.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.6.0
+        version: 1.4.6
 - !ruby/object:Gem::Dependency
-  name: gem_publisher
+  name: sitemap-parser
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
-  type: :development
+        version: 0.3.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -84,16 +112,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -138,7 +166,7 @@ dependencies:
         version: 1.18.0
 description: 
 email:
-- matt.bostock@digital.cabinet-office.gov.uk
+- govuk-dev@digital.cabinet-office.gov.uk
 executables:
 - seed-crawler
 extensions: []
@@ -148,14 +176,12 @@ files:
 - ".rspec"
 - ".ruby-version"
 - Gemfile
+- Jenkinsfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/seed-crawler
 - govuk_seed_crawler.gemspec
-- jenkins-branches.sh
-- jenkins-tests.sh
-- jenkins.sh
 - lib/govuk_seed_crawler.rb
 - lib/govuk_seed_crawler/amqp_client.rb
 - lib/govuk_seed_crawler/cli_parser.rb
@@ -170,7 +196,7 @@ files:
 - spec/govuk_seed_crawler/seeder_spec.rb
 - spec/integration/govuk_seed_crawler_spec.rb
 - spec/spec_helper.rb
-homepage: https://github.gds/gds/govuk_seed_crawler
+homepage: https://github.com/alphagov/govuk_seed_crawler
 licenses:
 - MIT
 metadata: {}
@@ -180,17 +206,16 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - "~>"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Retrieves a list of URLs to seed the crawler by publishing them to a RabbitMQ

data/jenkins-branches.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-[ -x .venv/bin/pip ] || virtualenv .venv
-. .venv/bin/activate
-
-pip install -q ghtools
-
-REPO="gds:gds/govuk_seed_crawler"
-gh-status "$REPO" "$GIT_COMMIT" pending -d "\"Build #${BUILD_NUMBER} is running on Jenkins\"" -u "$BUILD_URL" >/dev/null
-
-if ./jenkins-tests.sh; then
-  gh-status "$REPO" "$GIT_COMMIT" success -d "\"Build #${BUILD_NUMBER} succeeded on Jenkins\"" -u "$BUILD_URL" >/dev/null
-  exit 0
-else
-  gh-status "$REPO" "$GIT_COMMIT" failure -d "\"Build #${BUILD_NUMBER} failed on Jenkins\"" -u "$BUILD_URL" >/dev/null
-  exit 1
-fi

data/jenkins-tests.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-set -e
-rm -f Gemfile.lock
-bundle install --path "${HOME}/bundles/${JOB_NAME}"
-bundle exec rake
-bundle exec rake integration

data/jenkins.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-./jenkins-tests.sh
-bundle exec rake publish_gem