robots_tag_parser 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a809876ca8fb408908258b92c72b098ce6eb13256d5bbc96c1bea4bdca404bc3
+  data.tar.gz: ab449b96a869192412b0fd1e47cf5382cdfbff7c8a73ef104f958286b0a0616c
+SHA512:
+  metadata.gz: 7e963d9cc19af7ab621a17bc5fdc0a270acb7e7a8e2aa7ad73abfa225b5a8a60358a3783f998fbea560e999de79028c6ecf36ad973a5ecf2e5979a09ec71b0a3
+  data.tar.gz: 0f5704d0d9de04a9081ebe22e830da9cc8f353bdbcfad2f850b23484906e90f8f7a63db5376080df0d89a4726ea2e10248d38889ecdd1f88745586d37f56e735

data/.circleci/config.yml

@@ -0,0 +1,66 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2.1
+
+orbs:
+  ruby: circleci/ruby@2.0.0
+
+jobs:
+  build_and_test:
+    parameters:
+      ruby_version:
+        type: string
+    environment:
+      CC_TEST_REPORTER_ID: 04c5e730214438ccb56fa2e218bc65bcc1ae4ea86efb5c1f58f7eb9cc065c82a
+      COVERAGE: true
+    docker:
+      - image: cimg/ruby:<< parameters.ruby_version >>
+
+    working_directory: ~/robots_tag_parser
+
+    steps:
+      - checkout
+
+      # Install gems with Bundler
+      - ruby/install-deps:
+          # Need to clear the gem cache? Set or bump the CACHE_VERSION in your
+          # CircleCi project: Project Settings > Environment Variables
+          key: gems-ruby-<< parameters.ruby_version >>-v{{ .Environment.CACHE_VERSION }}
+
+      - run:
+          name: Prepare Code Climate Test Reporter
+          command: |
+            curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+            chmod +x ./cc-test-reporter
+            ./cc-test-reporter before-build
+
+      - ruby/rspec-test:
+          # Need to temporarily test with a particular seed? Use:
+          # order: rand:123
+          order: rand
+
+      - run:
+          name: Report Test Results
+          command: |
+            ./cc-test-reporter after-build
+
+      # collect reports
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+          destination: test-results
+
+workflows:
+  build_and_test:
+    jobs:
+      - build_and_test:
+          name: "Ruby << matrix.ruby_version >>"
+          matrix:
+            parameters:
+              ruby_version:
+                - 2.7.6
+                - 3.0.4
+                - 3.1.2

data/.codeinventory.yml

@@ -0,0 +1,13 @@
+name: RobotsTagParser
+description: 'A gem to parse X-Robots-Tag HTTP headers.'
+license: 'https://creativecommons.org/publicdomain/zero/1.0'
+openSourceProject: 1
+governmentWideReuseProject: 1
+tags:
+    - GSA
+    - Search.gov
+    - search
+    - indexing
+    - robots
+contact:
+    email: search@gsa.gov

data/.github/pull_request_template.md

@@ -0,0 +1,19 @@
+## Summary
+- Brief summary of the changes included in this PR
+- Any additional information or context which may help the reviewer
+ 
+### Checklist
+Please ensure you have addressed all concerns below before marking a PR "ready for review" or before requesting a re-review. If you cannot complete an item below, replace the checkbox with the ⚠️ `:warning:` emoji and explain why the step was not completed.
+ 
+#### Functionality Checks
+- [ ] You have merged the latest changes from the target branch (usually `main`) into your branch.
+ 
+- [ ] Your primary commit message is of the format **SRCH-#### \<description\>** matching the associated Jira ticket.
+ 
+- [ ] PR title is either of the format **SRCH-#### \<description\>** matching the associated Jira ticket (i.e. "SRCH-123 implement feature X"), or **Release - SRCH-####, SRCH-####, SRCH-####** matching the Jira ticket numbers in the release.
+ 
+- [ ] Automated checks pass, if applicable. If Code Climate checks do not pass, explain reason for failures:  
+
+#### Process Checks
+ 
+- [ ] You have specified at least one "Reviewer".

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+# Ignore the rubocop YAML file that is downloaded when running `rubocop` locally
+.rubocop*default-yml

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,2 @@
+inherit_from:
+  - https://raw.githubusercontent.com/GSA/searchgov_style/main/.default.yml

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in robots_tag_parser.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,111 @@
+PATH
+  remote: .
+  specs:
+    robots_tag_parser (0.1.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (7.0.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    ast (2.4.2)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.10)
+    debug (1.6.3)
+      irb (>= 1.3.6)
+      reline (>= 0.3.1)
+    diff-lcs (1.5.0)
+    docile (1.4.0)
+    i18n (1.12.0)
+      concurrent-ruby (~> 1.0)
+    io-console (0.5.11)
+    irb (1.4.2)
+      reline (>= 0.3.0)
+    method_source (1.0.0)
+    minitest (5.16.3)
+    parallel (1.22.1)
+    parser (3.1.2.1)
+      ast (~> 2.4.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rack (3.0.0)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.6.0)
+    reline (0.3.1)
+      io-console (~> 0.5)
+    rexml (3.2.5)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    rspec_junit_formatter (0.5.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    rubocop (1.31.0)
+      parallel (~> 1.10)
+      parser (>= 3.1.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.18.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.23.0)
+      parser (>= 3.1.1.0)
+    rubocop-performance (1.15.0)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    rubocop-rails (2.15.2)
+      activesupport (>= 4.2.0)
+      rack (>= 1.1)
+      rubocop (>= 1.7.0, < 2.0)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    rubocop-rspec (2.12.1)
+      rubocop (~> 1.31)
+    ruby-progressbar (1.11.0)
+    searchgov_style (0.1.20)
+      rubocop (= 1.31.0)
+      rubocop-performance (~> 1.9)
+      rubocop-rails (~> 2.9)
+      rubocop-rake (~> 0.5)
+      rubocop-rspec (~> 2.5)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    tzinfo (2.0.5)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 2.3)
+  debug
+  pry
+  rake
+  robots_tag_parser!
+  rspec (~> 3.0)
+  rspec_junit_formatter
+  searchgov_style
+  simplecov
+
+BUNDLED WITH
+   2.3.15

data/LICENSE.md

@@ -0,0 +1,31 @@
+As a work of the United States government, this project is in the
+public domain within the United States.
+
+Additionally, we waive copyright and related rights in the work
+worldwide through the CC0 1.0 Universal public domain dedication.
+
+## CC0 1.0 Universal summary
+
+This is a human-readable summary of the [Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode).
+
+### No copyright
+
+The person who associated a work with this deed has dedicated the work to
+the public domain by waiving all rights to the work worldwide
+under copyright law, including all related and neighboring rights, to the
+extent allowed by law.
+
+You can copy, modify, distribute and perform the work, even for commercial
+purposes, all without asking permission.
+
+### Other information
+
+In no way are the patent or trademark rights of any person affected by CC0,
+nor are the rights that other persons may have in the work or in how the
+work is used, such as publicity or privacy rights.
+
+Unless expressly stated otherwise, the person who associated a work with
+this deed makes no warranties about the work, and disclaims liability for
+all uses of the work, to the fullest extent permitted by applicable law.
+When using or citing the work, you should not imply endorsement by the
+author or the affirmer.

data/README.md

@@ -0,0 +1,69 @@
+# RobotsTagParser
+
+A simple gem to parse X-Robots-Tag HTTP headers according to [Google X-Robots-Tag HTTP header specifications](https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag#using-the-x-robots-tag-http-header).
+
+[![CircleCI](https://circleci.com/gh/GSA/robots_tag_parser.svg?style=shield&circle-token=:circle-token)](https://circleci.com/gh/GSA/robots_tag_parser)
+[![Code Climate](https://codeclimate.com/github/GSA/robots_tag_parser.svg)](https://codeclimate.com/github/GSA/robots_tag_parser)
+[![Test Coverage](https://codeclimate.com/github/GSA/robots_tag_parser/badges/coverage.svg)](https://codeclimate.com/github/GSA/robots_tag_parser/coverage)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'robots_tag_parser', git: 'https://github.com/GSA/robots_tag_parser'
+```
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+### Basic examples
+Get rules applying to all user agents:
+
+```ruby
+headers = { 'X-Robots-Tag' => ['noindex,noarchive', 'googlebot: nofollow'] }
+
+RobotsTagParser.get_rules(headers: headers)
+=> ['noindex', 'noarchive']
+```
+
+Get rules applying to specific user agents (which include generic
+rules):
+
+```ruby
+headers = { 'X-Robots-Tag' => ['noindex,noarchive', 'googlebot: nofollow'] }
+
+RobotsTagParser.get_rules(headers: headers, user_agent: 'googlebot')
+=> ['noindex', 'noarchive', 'nofollow']
+```
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on [GitHub](https://github.com/GSA/robots_tag_parser).
+
+## License
+
+The gem is available as open source under the terms of the [CC0 License](LICENSE.md).
+
+## Directives:
+- [x] `all` - There are no restrictions for indexing or serving.
+- [x] `none` - Equivalent to `noindex` and `nofollow`.
+- [x] `noindex` - Do not show this page in search results and do not show a "Cached" link in search results.
+- [x] `nofollow` - Do not follow the links on this page.
+- [x] `noarchive` - Do not show a "Cached" link in search results.
+- [x] `nosnippet` - Do not show a snippet in the search results for this page.
+- [x] `notranslate` - Do not offer translation of this page in search results.
+- [x] `noimageindex` - Do not index images on this page.
+- [x] `unavailable_after` - Do not show this page in search results after the specified date/time.
+
+Source: [https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag](https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag#valid-indexing--serving-directives)

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "robots_tag_parser"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/robots_tag_parser/version.rb

@@ -0,0 +1,3 @@
+module RobotsTagParser
+  VERSION = "0.1.1"
+end

data/lib/robots_tag_parser.rb

@@ -0,0 +1,30 @@
+require "robots_tag_parser/version"
+
+module RobotsTagParser
+  def self.get_rules(headers:, user_agent: nil)
+    raise ArgumentError, 'Headers must be a hash' unless headers.is_a? Hash
+    rules = downcase_keys(headers)['x-robots-tag']
+    return [] unless rules
+    hash_of(rules).values_at('all', user_agent).compact.flatten.uniq
+  end
+
+  def self.downcase_keys(hash)
+    new_hash = {}
+    hash.each_pair{ |key, val| new_hash[key.downcase] = val }
+    new_hash
+  end
+
+  def self.hash_of(rules)
+    rules = (rules.is_a? String) ? [rules] : rules
+    rules.map!{ |rule| RuleParser.parse(rule) }.reduce(&:merge)
+  end
+
+  class RuleParser
+    def self.parse(rule_string)
+      rule = rule_string.downcase.split(/: */)
+      user_agent, rules = (rule.length == 1 ? 'all' : rule[0]), rule.last
+      rules = rules.split(',').map(&:strip)
+      Hash[[[user_agent, rules]]]
+    end
+  end
+end

data/robots_tag_parser.gemspec

@@ -0,0 +1,31 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "robots_tag_parser/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "robots_tag_parser"
+  spec.version       = RobotsTagParser::VERSION
+  spec.authors       = ["Martha Thompson"]
+  spec.email         = ["mothonmars@gmail.com"]
+
+  spec.summary       = %q{X-Robots-Tag HTTP header parser}
+  spec.description   = %q{A simple gem to parse X-Robots-Tag HTTP headers according to Google X-Robots-Tag HTTP header specifications.}
+  spec.homepage      = "https://github.com/GSA/robots_tag_parser"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", ">= 2.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rspec_junit_formatter" # used in CircleCI
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "debug"
+  spec.add_development_dependency "searchgov_style"
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: robots_tag_parser
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Martha Thompson
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-11-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: searchgov_style
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple gem to parse X-Robots-Tag HTTP headers according to Google X-Robots-Tag
+  HTTP header specifications.
+email:
+- mothonmars@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".codeinventory.yml"
+- ".github/pull_request_template.md"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.md
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/robots_tag_parser.rb
+- lib/robots_tag_parser/version.rb
+- robots_tag_parser.gemspec
+homepage: https://github.com/GSA/robots_tag_parser
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.15
+signing_key:
+specification_version: 4
+summary: X-Robots-Tag HTTP header parser
+test_files: []