govuk_seed_crawler 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a37f428070681bc4ca2466497df0b69f45fed94a
-  data.tar.gz: 55bdafe5ade9251f6f630eeb490e481a9796fc4f
+  metadata.gz: 0f12b494cb9d0b5cb48495917d02f70caaec3e19
+  data.tar.gz: 58db84b76c22e4a80dee1c7528a8b02e59e40bd5
 SHA512:
-  metadata.gz: 631f38d96a7d1ea301b38e761d5c55debbb9ee0c99a8e2a88ef1bb965b12637eceaaba70dfbd13941e4b96c088781fc6da3f724e283bfe8afc6d3dd8f0732321
-  data.tar.gz: 2f8b41afecdaba199b32925b13804c6dce588535b611a01383753dac2a22a655e728a9b77cf529845e0b6df1738bdca716c0eac5a5a4032e70e61d7d94c9cd82
+  metadata.gz: 6911991e1987cae4f9510c60988cd096339121257cfa2ee6ee585494e97e2ae660b692485c18a9efccfb300fd30f7d9206f76d65623b0f80ab43b7b99add35e1
+  data.tar.gz: 39befcb79fe0c2123b0047adb79292fc8b4d971d85f077d1f0e587538578c59d035e379317fab6e269b9dc3322d9a69892ab5a37d5e62c0c84e74b53e75515b9

data/README.md

@@ -1,6 +1,7 @@
 # GOV.UK: Seed the Crawler
 
-Retrieves a list of URLs to seed the [crawler](https://github.com/alphagov/govuk_crawler_worker) by publishing them to a RabbitMQ exchange.
+This gem retrieves a list of seed URLs from the GOV.UK sitemap and adds them to RabbitMQ
+so that the [crawler](https://github.com/alphagov/govuk_crawler_worker) can consume them.
 
 ## Installation
 

data/govuk_seed_crawler.gemspec

@@ -6,10 +6,10 @@ require 'govuk_seed_crawler/version'
 Gem::Specification.new do |spec|
   spec.name          = "govuk_seed_crawler"
   spec.version       = GovukSeedCrawler::VERSION
-  spec.authors       = ["Matt Bostock"]
-  spec.email         = ["matt.bostock@digital.cabinet-office.gov.uk"]
+  spec.authors       = ['GOV.UK developers']
+  spec.email         = ["govuk-dev@digital.cabinet-office.gov.uk"]
   spec.summary       = %q{Retrieves a list of URLs to seed the crawler by publishing them to a RabbitMQ exchange.}
-  spec.homepage      = "https://github.gds/gds/govuk_seed_crawler"
+  spec.homepage      = "https://github.com/alphagov/govuk_seed_crawler"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "bunny", "~> 1.3"
-  spec.add_runtime_dependency "govuk_mirrorer", "~> 1.3.1"
+  spec.add_runtime_dependency "sitemap-parser", "~> 0.3.0"
   spec.add_runtime_dependency "slop", "~> 3.6.0"
 
   spec.add_development_dependency "gem_publisher", "~> 1.3"

data/jenkins-branches.sh

@@ -6,7 +6,7 @@ set -e
 
 pip install -q ghtools
 
-REPO="gds:gds/govuk_seed_crawler"
+REPO="alphagov/govuk_seed_crawler"
 gh-status "$REPO" "$GIT_COMMIT" pending -d "\"Build #${BUILD_NUMBER} is running on Jenkins\"" -u "$BUILD_URL" >/dev/null
 
 if ./jenkins-tests.sh; then

data/lib/govuk_seed_crawler/indexer.rb

@@ -1,5 +1,4 @@
-require 'govuk_mirrorer/indexer'
-require 'govuk_mirrorer/statsd'
+require 'sitemap-parser'
 
 module GovukSeedCrawler
   class Indexer
@@ -9,8 +8,9 @@ module GovukSeedCrawler
       raise "No site_root defined" unless site_root
 
       GovukSeedCrawler.logger.info("Retrieving list of URLs for #{site_root}")
-      indexer = GovukMirrorer::Indexer.new(site_root)
-      @urls = indexer.all_start_urls
+
+      sitemap = SitemapParser.new("#{site_root}/sitemap.xml", {recurse: true})
+      @urls = sitemap.to_a
 
       GovukSeedCrawler.logger.info("Found #{@urls.count} URLs")
     end

data/lib/govuk_seed_crawler/version.rb

@@ -1,3 +1,3 @@
 module GovukSeedCrawler
-  VERSION = "1.0.0"
+  VERSION = "2.0.0"
 end

data/spec/govuk_seed_crawler/indexer_spec.rb

@@ -1,20 +1,20 @@
 require 'spec_helper'
 
 describe GovukSeedCrawler::Indexer do
-  subject { GovukSeedCrawler::Indexer.new('https://example.com/') }
+  subject { GovukSeedCrawler::Indexer.new('https://example.com') }
 
   context "under normal usage" do
-    let(:mock_indexer) do
-      double(:mock_indexer, :all_start_urls => [])
+    let(:mock_parser) do
+      double(:mock_parser, :to_a => [])
     end
 
     it "responds to Indexer#urls" do
-      allow(GovukMirrorer::Indexer).to receive(:new).and_return(mock_indexer)
+      allow(SitemapParser).to receive(:new).and_return(mock_parser)
       expect(subject).to respond_to(:urls)
     end
 
-    it "calls GovukMirrorer::Indexer with the site root" do
-      expect(GovukMirrorer::Indexer).to receive(:new).with('https://example.com/').and_return(mock_indexer)
+    it "calls SitemapParser with the sitemap file" do
+      expect(SitemapParser).to receive(:new).with('https://example.com/sitemap.xml', {:recurse => true}).and_return(mock_parser)
       subject
     end
   end

data/spec/integration/govuk_seed_crawler_spec.rb

@@ -2,36 +2,30 @@ require 'json'
 require 'spec_helper'
 
 describe GovukSeedCrawler do
-  def stub_api_artefacts(count)
-    item = {
-      "id" => "https://www.gov.uk/api/government%2Fnews%2Ffaster-review-of-support-for-renewable-electricity-to-provide-investor-certainty.json",
-      "web_url" => "https://www.gov.uk/government/news/faster-review-of-support-for-renewable-electricity-to-provide-investor-certainty",
-      "title" => "Faster review of support for Renewable electricity to provide investor certainty",
-      "format" => "announcement"
-    }
-    results = count.times.collect { item }
-    response = {
-      "_response_info" => {
-        "status" => "ok",
-        "links" => []
-      },
-      "total" => results.size,
-      "start_index" => 1,
-      "page_size" => 100,
-      "current_page" => 1,
-      "pages" => 1,
-      "results" => results
+  def stub_sitemap
+    sitemap = %{<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>https://www.gov.uk/</loc>
+  </url>
+  <url>
+    <loc>https://www.gov.uk/register-to-vote</loc>
+  </url>
+  <url>
+    <loc>https://www.gov.uk/help</loc>
+  </url>
+</urlset>
     }
 
-    stub_request(:get, "https://www.gov.uk//api/artefacts.json").
-         to_return(:status => 200, :body => response.to_json, :headers => {})
+    stub_request(:get, "https://www.gov.uk/sitemap.xml").
+         to_return(:status => 200, :body => sitemap, :headers => {})
   end
 
   let(:vhost) { "/" }
   let(:exchange_name) { "govuk_seed_crawler_integration_exchange" }
   let(:queue_name) { "govuk_seed_crawler_integration_queue" }
   let(:topic) { "#" }
-  let(:site_root) { "https://www.gov.uk/" }
+  let(:site_root) { "https://www.gov.uk" }
   let(:options) {{
       :host => ENV.fetch("AMQP_HOST", "localhost"),
       :user => ENV.fetch("AMQP_USER", "govuk_seed_crawler"),
@@ -57,10 +51,9 @@ describe GovukSeedCrawler do
   end
 
   it "publishes URLs it finds to an AMQP topic exchange" do
-    stub_api_artefacts(10)
+    stub_sitemap
     subject
 
-    # There's an extra 5 URLs from the Indexer class that are hard-coded.
-    expect(@queue.message_count).to be(15)
+    expect(@queue.message_count).to be(3)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: govuk_seed_crawler
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
-- Matt Bostock
+- GOV.UK developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-28 00:00:00.000000000 Z
+date: 2016-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -25,19 +25,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: govuk_mirrorer
+  name: sitemap-parser
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: 0.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
@@ -138,7 +138,7 @@ dependencies:
         version: 1.18.0
 description: 
 email:
-- matt.bostock@digital.cabinet-office.gov.uk
+- govuk-dev@digital.cabinet-office.gov.uk
 executables:
 - seed-crawler
 extensions: []
@@ -170,7 +170,7 @@ files:
 - spec/govuk_seed_crawler/seeder_spec.rb
 - spec/integration/govuk_seed_crawler_spec.rb
 - spec/spec_helper.rb
-homepage: https://github.gds/gds/govuk_seed_crawler
+homepage: https://github.com/alphagov/govuk_seed_crawler
 licenses:
 - MIT
 metadata: {}