sitemaps_parser 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7ba72de307e8ad869330f8577085aeb085f9d374
-  data.tar.gz: bd1a8740911500318dd227ee311fa31f2e457f81
+SHA256:
+  metadata.gz: dd9e770cf06954a14d869600cb2c6281c7ee993f2d29d628abb1cf5a5b4809f1
+  data.tar.gz: bea190bd461d7eec0e72f092ae56546dce1b4fc2568aa2dd4757abf9b4675067
 SHA512:
-  metadata.gz: 8b504a926a9e51611258f93ba88a85db5664e0939f6bc5337599887b6f258738d8560140f20521bbc2698b0322ec2e3d67abfbd57d02f01258875954f6664a5a
-  data.tar.gz: 2a92c0b888363b473a1cb3d981c3533b4fafe15eacdf7c920aa35e131c8a4a9db2284cef13958b73e52886110c8556b61acf61a6d7272c1e3c375434540a702d
+  metadata.gz: d4ee0f7fcd11e03a9f1a031088a0171b6b13e6a2c6bdbba1b9aac1232fe526a1947f7bcabf402714b071efa6fee5561fb413e154b8558d87d77d39fea8932117
+  data.tar.gz: 6bfee33a31ecb2680eabe09a9bf39ca19fb8c72e2f45af78275217f9898793039090c7352c7fba0b041331a15ef74fe2540ce200980168b2dfaf346ee24dc1f1

data/.circleci/config.yml

@@ -0,0 +1,75 @@
+version: 2.1
+
+executors:
+  test_executor:
+    working_directory: ~/sitemaps
+
+    docker:
+      - image: circleci/ruby:${RUBY_VERSION}
+
+jobs:
+  build:
+    environment:
+      RUBY_VERSION: << parameters.ruby_version >>
+      ACTIVESUPPORT_VERSION: << parameters.activesupport_version >>
+    executor: test_executor
+    parameters:
+      ruby_version:
+        type: string
+      activesupport_version:
+        type: string
+        default: '~> 5.0'
+    steps:
+      - checkout
+
+      - restore_cache:
+          keys:
+            - sitemaps-cache-v1-{{ arch }}-{{ .Branch }}-{{ .Revision }}
+            - sitemaps-cache-v1-{{ arch }}-{{ .Branch }}
+            - sitemaps-cache-v1
+
+      - run:
+          name: Bundle Install
+          command: |
+            bundle check --path=vendor/bundle || bundle install --clean --path vendor/bundle
+
+      - save_cache:
+          key: sitemaps-cache-v1-{{ arch }}-{{ .Branch }}-{{ .Revision }}
+          paths:
+            - vendor/bundle
+
+      - run:
+          name: RSpec
+          command: bundle exec rspec spec
+
+      # Once lower-level offenses are resolved, the fail-level flags should be removed
+      # so that any offense will cause the build to fail.
+      - run:
+          name: Rubocop
+          command: bundle exec rubocop --fail-level warning --display-only-fail-level-offenses
+
+workflows:
+  build_and_test:
+    jobs:
+      - build:
+          name: 'ruby 2.3.8'
+          ruby_version: 2.3.8
+      - build:
+          name: 'ruby 2.4.6'
+          ruby_version: 2.4.6
+      - build:
+          name: 'ruby 2.5.5'
+          ruby_version: 2.5.5
+      - build:
+          name: 'ruby 2.6.3, activesupport 4'
+          ruby_version: 2.6.3
+          activesupport_version: '4.0.0'
+      - build:
+          name: 'ruby 2.6.3, activesupport 5'
+          ruby_version: 2.6.3
+          activesupport_version: '~> 5.0'
+      - build:
+          name: 'ruby 2.6.3, activesupport 6'
+          ruby_version: 2.6.3
+          activesupport_version: '6.0.0.rc1'
+

data/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.2.3] - 2019-06-19
+
+### Changed
+- [Run specs on CircleCi instead of Travis](https://github.com/GSA/sitemaps/commit/cd999ecbc6bcee36c20553c2d01b3bf1a24f4fad)
+- [Update gem metadata](https://github.com/GSA/sitemaps/commit/67603d415af3deb989eb2d6f85be2f64461a9be5)
+- [Require 100% code coverage](https://github.com/GSA/sitemaps/commit/032804a6476109150f10672e623b3412eb9dda11)
+- [Depend on activesupport >4, < 7](https://github.com/GSA/sitemaps/commit/02ff67f8e4e5470942105b5dbe0b90bbdbaa5176)
+
+### Fixed
+- [Parse sitemaps including extra whitespace](https://github.com/GSA/sitemaps/commit/1ed7a427eb21a4a37d41b9cbfdfd81107e109c76)
+- [Discover commented sitemaps in robots.txt](https://github.com/GSA/sitemaps/commit/2d3bd84140dc0df15112eb1e4f1f3b27d2ac6224)

data/LICENSE.md

@@ -0,0 +1,31 @@
+As a work of the United States government, this project is in the
+public domain within the United States.
+
+Additionally, we waive copyright and related rights in the work
+worldwide through the CC0 1.0 Universal public domain dedication.
+
+## CC0 1.0 Universal summary
+
+This is a human-readable summary of the [Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode).
+
+### No copyright
+
+The person who associated a work with this deed has dedicated the work to
+the public domain by waiving all rights to the work worldwide
+under copyright law, including all related and neighboring rights, to the
+extent allowed by law.
+
+You can copy, modify, distribute and perform the work, even for commercial
+purposes, all without asking permission.
+
+### Other information
+
+In no way are the patent or trademark rights of any person affected by CC0,
+nor are the rights that other persons may have in the work or in how the
+work is used, such as publicity or privacy rights.
+
+Unless expressly stated otherwise, the person who associated a work with
+this deed makes no warranties about the work, and disclaims liability for
+all uses of the work, to the fullest extent permitted by applicable law.
+When using or citing the work, you should not imply endorsement by the
+author or the affirmer.

data/README.md

@@ -1,10 +1,11 @@
 # Sitemaps
 
 [![Gem](https://img.shields.io/gem/v/sitemaps_parser.svg?style=flat-square)](https://rubygems.org/gems/sitemaps_parser)
+[![CircleCI](https://circleci.com/gh/GSA/sitemaps.svg?style=svg)](https://circleci.com/gh/GSA/sitemaps)
 
 Discover, retrieve and parse XML sitemaps, according to the spec at [sitemaps.org](http://sitemaps.org).
 
-See [RDOC Documentation](http://lygaret.github.io/sitemaps) for detailed documentation.
+See [RDOC Documentation](https://gsa.github.io/sitemaps) for detailed documentation.
 
 ## Installation
 
@@ -56,7 +57,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/lygaret/sitemaps. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/GSA/sitemaps. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## License
 

data/lib/sitemaps.rb

@@ -1,14 +1,15 @@
-require "active_support"
-require "active_support/core_ext/object/try"
-require "active_support/core_ext/object/blank"
+require 'active_support'
+require 'active_support/core_ext/object/try'
+require 'active_support/core_ext/object/blank'
 
-require "set"
-require "time"
-require "rexml/document"
+require 'set'
+require 'time'
+require 'rexml/document'
+require 'net/http'
 
-require "sitemaps/version"
-require "sitemaps/parser"
-require "sitemaps/fetcher"
+require 'sitemaps/version'
+require 'sitemaps/parser'
+require 'sitemaps/fetcher'
 
 # Discover, fetch and parse XML sitemaps as defined by the `http://sitemaps.org` spec.
 module Sitemaps
@@ -168,11 +169,13 @@ module Sitemaps
     def discover_roots(url, fetcher)
       robots = begin
         robotsurl      = url.clone
-        robotsurl.path = "/robots.txt"
+        robotsurl.path = '/robots.txt'
         robotstxt      = fetcher.call(robotsurl)
 
-        discovered = robotstxt.scan(/^Sitemap: (.+)$/).flatten.reverse.map { |u| URI.parse(u.strip) }
-        discovered.empty? ? nil : discovered
+        discovered = robotstxt.scan(/^Sitemap: (\S+)/).flatten.map do |url|
+          URI.parse(url.strip)
+        end
+        discovered.presence
       rescue
         nil
       end

data/lib/sitemaps/parser.rb

@@ -40,29 +40,33 @@ module Sitemaps
     # @api private
     # @private
     def self.parse_loc(root)
-      loc = root.get_text("loc").try(:value)
+      loc = get_text(root, 'loc')
       loc && URI.parse(loc) rescue nil
     end
 
     # @api private
     # @private
     def self.parse_lastmod(root)
-      mod = root.get_text("lastmod").try(:value)
+      mod = get_text(root, 'lastmod')
       mod && Time.parse(mod) rescue nil
     end
 
     # @api private
     # @private
     def self.parse_changefreq(root)
-      freq = root.get_text("changefreq").try(:value)
+      freq = get_text(root, 'changefreq')
       freq && VALID_CHANGEFREQ.include?(freq) ? freq.to_sym : nil
     end
 
     # @api private
     # @private
     def self.parse_priority(root)
-      priority = root.get_text("priority").try(:value) || "0.5"
+      priority = get_text(root, 'priority') || '0.5'
       priority && Float(priority) rescue 0.5 # default priority according to spec
     end
+
+    def self.get_text(root, key)
+      root.get_text(key)&.value&.strip
+    end
   end
 end

data/lib/sitemaps/version.rb

@@ -1,3 +1,3 @@
 module Sitemaps
-  VERSION = "0.2.2".freeze
+  VERSION = "0.2.3".freeze
 end

data/sitemaps.gemspec

@@ -4,29 +4,31 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'sitemaps/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "sitemaps_parser"
+  spec.name          = 'sitemaps_parser'
   spec.version       = Sitemaps::VERSION
-  spec.authors       = ["Jonathan Raphaelson"]
-  spec.email         = ["jraphaelson@termscout.com"]
+  spec.authors       = ['Jonathan Raphaelson']
+  spec.email         = ['jraphaelson@termscout.com']
 
-  spec.summary       = "Retrieve and parse sitemaps, according to the sitemaps.org spec."
-  spec.homepage      = "http://github.com/termscout/sitemaps"
-  spec.license       = "MIT"
+  spec.summary       = 'Retrieve and parse sitemaps, according to the sitemaps.org spec.'
+  spec.homepage      = 'http://github.com/GSA/sitemaps'
+  spec.license       = 'CC0 1.0 Universal'
 
   files = `git ls-files -z`.split("\x0")
   files.reject! { |f| f.match(%r{^(test|spec|features)/}) }
 
   spec.files         = files
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 1"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "webmock", "~> 1"
-  spec.add_development_dependency "vcr", "~> 3"
-  spec.add_development_dependency "rubocop", "~> 0.38.0"
-  spec.add_development_dependency "byebug", "~> 8.2"
-  spec.add_development_dependency "yard", "~> 0.8"
+  spec.add_development_dependency 'bundler', '~> 1'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'webmock', '~> 3.0'
+  spec.add_development_dependency 'vcr', '~> 3'
+  spec.add_development_dependency 'rubocop', '~> 0.71.0'
+  spec.add_development_dependency 'byebug', '~> 8.2'
+  spec.add_development_dependency 'yard', '~> 0.9.11'
+  spec.add_development_dependency 'simplecov', '~> 0.16'
 
-  spec.add_runtime_dependency "activesupport", "~> 4"
+  spec.add_runtime_dependency 'activesupport',
+    (ENV['ACTIVESUPPORT_VERSION'] || ['>= 4', '< 7'])
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sitemaps_parser
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Jonathan Raphaelson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-25 00:00:00.000000000 Z
+date: 2019-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.38.0
+        version: 0.71.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.38.0
+        version: 0.71.0
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -114,28 +114,48 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 0.9.11
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 0.9.11
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 description: 
 email:
 - jraphaelson@termscout.com
@@ -143,13 +163,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
-- LICENSE.txt
+- LICENSE.md
 - README.md
 - Rakefile
 - bin/console
@@ -159,9 +180,9 @@ files:
 - lib/sitemaps/parser.rb
 - lib/sitemaps/version.rb
 - sitemaps.gemspec
-homepage: http://github.com/termscout/sitemaps
+homepage: http://github.com/GSA/sitemaps
 licenses:
-- MIT
+- CC0 1.0 Universal
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -178,10 +199,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Retrieve and parse sitemaps, according to the sitemaps.org spec.
 test_files: []
-has_rdoc: 

data/.travis.yml

@@ -1,4 +0,0 @@
-language: ruby
-rvm:
-  - 2.3.0
-before_install: gem install bundler -v 1.11.2

data/LICENSE.txt

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2016 TermScout
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.