gman 7.0.6 → 8.0.0

data/.github/workflows/codeql-analysis.yml

@@ -1,70 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ main ]
-  schedule:
-    - cron: '38 17 * * 5'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'ruby' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://git.io/codeql-language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2

data/.github/workflows/validate.yml

@@ -1,30 +0,0 @@
-name: "Validate Domain List"
-
-on:
-  schedule:
-    - cron: '0 0 15 * *'
-  workflow_dispatch: {}
-
-jobs:
-  validate: 
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Ruby
-
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3.0
-          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-
-      - name: Validate domains
-        run: bundle exec script/validate-domains
- 
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v4
-        if: always()
-        with:
-            commit-message: Remove invalid domains
-            title: Remove invalid domains
-            add-paths: config/*

data/.github/workflows/vendor.yml

@@ -1,29 +0,0 @@
-name: "Vendor Domain List"
-
-on:
-  schedule:
-    - cron: '0 0 1 * * '
-  workflow_dispatch: {}
-
-jobs:
-  vendor: 
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Ruby
-
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3.0
-          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-
-      - name: Vendor
-        run: script/vendor
-
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v4
-        with:
-            commit-message: Vendor domain list
-            title: Vendor domain list
-            add-paths: config/*

data/.gitignore

@@ -1,6 +0,0 @@
-*.gem
-.bundle
-tmp
-Gemfile.lock
-/almanak.overheid.nl
-spec/examples.txt

data/.rspec

@@ -1,2 +0,0 @@
---color
---require spec_helper

data/.rubocop.yml

@@ -1,29 +0,0 @@
-inherit_from: .rubocop_todo.yml
-
-require:
-  - rubocop-performance
-  - rubocop-rspec
-
-AllCops:
-  NewCops: enable
-  TargetRubyVersion: 2.5
-  Exclude:
-    - config/**/*
-    - vendor/**/*
-
-Style/Documentation:
-  Enabled: false
-
-Metrics/AbcSize:
-  Exclude:
-    - lib/gman/importer.rb
-
-Metrics/ClassLength:
-  Enabled: false
-
-Metrics/BlockLength:
-  Enabled: false
-
-Style/AccessModifierDeclarations:
-  Exclude:
-    - lib/gman/identifier.rb

data/.rubocop_todo.yml

@@ -1,84 +0,0 @@
-# This configuration was generated by
-# `rubocop --auto-gen-config`
-# on 2020-11-13 00:12:15 UTC using RuboCop version 1.3.0.
-# The point is for the user to remove these configuration records
-# one by one as the offenses are removed from the code base.
-# Note that changes in the inspected code, or installation of new
-# versions of RuboCop, may require this file to be generated again.
-
-# Offense count: 1
-# Configuration parameters: MinSize.
-Performance/CollectionLiteralInLoop:
-  Exclude:
-    - 'spec/gman/domains_spec.rb'
-
-# Offense count: 40
-# Configuration parameters: Prefixes.
-# Prefixes: when, with, without
-RSpec/ContextWording:
-  Exclude:
-    - 'spec/gman/bin_spec.rb'
-    - 'spec/gman/domain_list_spec.rb'
-    - 'spec/gman/identifier_spec.rb'
-    - 'spec/gman/importer_spec.rb'
-    - 'spec/gman/locality_spec.rb'
-    - 'spec/gman_spec.rb'
-
-# Offense count: 4
-# Configuration parameters: IgnoredMetadata.
-RSpec/DescribeClass:
-  Exclude:
-    - 'spec/gman/bin_spec.rb'
-    - 'spec/gman/country_code_spec.rb'
-    - 'spec/gman/domains_spec.rb'
-    - 'spec/gman/identifier_spec.rb'
-
-# Offense count: 3
-# Configuration parameters: Max.
-RSpec/ExampleLength:
-  Exclude:
-    - 'spec/gman/bin_spec.rb'
-    - 'spec/gman/country_code_spec.rb'
-    - 'spec/gman/domains_spec.rb'
-
-# Offense count: 2
-# Configuration parameters: AssignmentOnly.
-RSpec/InstanceVariable:
-  Exclude:
-    - 'spec/gman/importer_spec.rb'
-
-# Offense count: 23
-RSpec/MultipleExpectations:
-  Max: 3
-
-# Offense count: 24
-# Configuration parameters: AllowSubject.
-RSpec/MultipleMemoizedHelpers:
-  Max: 9
-
-# Offense count: 81
-# Configuration parameters: IgnoreSharedExamples.
-RSpec/NamedSubject:
-  Exclude:
-    - 'spec/gman/country_code_spec.rb'
-    - 'spec/gman/domain_list_spec.rb'
-    - 'spec/gman/identifier_spec.rb'
-    - 'spec/gman/importer_spec.rb'
-    - 'spec/gman_spec.rb'
-
-# Offense count: 10
-RSpec/NestedGroups:
-  Max: 5
-
-# Offense count: 2
-RSpec/ScatteredSetup:
-  Exclude:
-    - 'spec/gman/importer_spec.rb'
-
-# Offense count: 4
-Security/Open:
-  Exclude:
-    - 'script/reconcile-us'
-    - 'script/vendor-federal-de'
-    - 'script/vendor-municipal-de'
-    - 'script/vendor-us'

data/Gemfile

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-source 'https://rubygems.org'
-
-gemspec

data/docs/CODE_OF_CONDUCT.md

@@ -1,46 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at ben@balter.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/

data/docs/CONTRIBUTING.md

@@ -1,92 +0,0 @@
-# Contributing to Gman
-
-Hi there! We're thrilled that you'd like to contribute to Gman. Your help is essential for keeping it great.
-
-Gman is an open source project supported by the efforts of an entire community and built one contribution at a time by users like you. We'd love for you to get involved. Whatever your level of skill or however much time you can give, your contribution is greatly appreciated. There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests, helping other users by commenting on issues, or writing code which can be incorporated into Gman itself.
-
-Following these guidelines helps to communicate that you respect the time of the developers managing and developing this open source project. In return, they should reciprocate that respect in addressing your issue, assessing changes, and helping you finalize your pull requests.
-
-
-
-## How to report a bug
-
-Think you found a bug? Please check [the list of open issues](https://github.com/benbalter/gman/issues) to see if your bug has already been reported. If it hasn't please [submit a new issue](https://github.com/benbalter/gman/issues/new).
-
-Here are a few tips for writing *great* bug reports:
-
-* Describe the specific problem (e.g., "widget doesn't turn clockwise" versus "getting an error")
-* Include the steps to reproduce the bug, what you expected to happen, and what happened instead
-* Check that you are using the latest version of the project and its dependencies
-* Include what version of the project your using, as well as any relevant dependencies
-* Only include one bug per issue. If you have discovered two bugs, please file two issues
-* Include screenshots or screencasts whenever possible
-* Even if you don't know how to fix the bug, including a failing test may help others track it down
-
-**If you find a security vulnerability, do not open an issue. Please email ben@balter.com instead.**
-
-## How to suggest a feature or enhancement
-
-If you find yourself wishing for a feature that doesn't exist in Gman, you are probably not alone. There are bound to be others out there with similar needs. Many of the features that Gman has today have been added because our users saw the need.
-
-Feature requests are welcome. But take a moment to find out whether your idea fits with the scope and goals of the project. It's up to you to make a strong case to convince the project's developers of the merits of this feature. Please provide as much detail and context as possible, including describing the problem you're trying to solve.
-
-[Open an issue](https://github.com/benbalter/gman/issues/new) which describes the feature you would like to see, why you want it, how it should work, etc.
-
-## Domains
-
-Domains live in [`config/domains.txt`](../config/domains.txt) as a list of TLDs and SLD+TLDs.
-
-Right now, the only valid government top level domains (TLDs), represent the US government and are `.gov`, and `.mil`. Secondary domains (e.g., `gov.uk`, or `mil.au`) represent non-US government entities.
-
-To add or remove a domain from the list of known government domains, simply edit the `domains.txt` file.
-
-
-## Your first contribution
-
-We'd love for you to contribute to the project. Unsure where to begin contributing to Gman? You can start by looking through these "good first issue" and "help wanted" issues:
-
-* [Good first issues](https://github.com/benbalter/gman/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - issues which should only require a few lines of code and a test or two
-* [Help wanted issues](https://github.com/benbalter/gman/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) - issues which may be a bit more involved, but are specifically seeking community contributions
-
-*p.s. Feel free to ask for help; everyone is a beginner at first* :smiley_cat:
-
-## How to propose changes
-
-Here's a few general guidelines for proposing changes:
-
-* If you are changing any user-facing functionality, please be sure to update the documentation
-* If you are adding a new behavior or changing an existing behavior, please be sure to update the corresponding test(s)
-* Each pull request should implement **one** feature or bug fix. If you want to add or fix more than one thing, submit more than one pull request
-* Do not commit changes to files that are irrelevant to your feature or bug fix
-* Don't bump the version number in your pull request (it will be bumped prior to release)
-* Write [a good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)
-
-At a high level, [the process for proposing changes](https://guides.github.com/introduction/flow/) is:
-
-1. [Fork](https://github.com/benbalter/gman/fork) and clone the project
-2. Configure and install the dependencies: `script/bootstrap`
-3. Make sure the tests pass on your machine: `script/cibuild`
-4. Create a descriptively named branch: `git checkout -b my-branch-name`
-5. Make your change, add tests and documentation, and make sure the tests still pass
-6. Push to your fork and [submit a pull request](https://github.com/benbalter/gman/compare) describing your change
-7. Pat your self on the back and wait for your pull request to be reviewed and merged
-
-**Interesting in submitting your first Pull Request?** It's easy! You can learn how from this *free* series [How to Contribute to an Open Source Project on GitHub](https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github)
-
-## Bootstrapping your local development environment
-
-`script/bootstrap`
-
-## Running tests
-
-`script/cibuild`
-
-## Code of conduct
-
-This project is governed by [the Contributor Covenant Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code.
-
-## Additional Resources
-
-* [Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/)
-* [Using Pull Requests](https://help.github.com/articles/using-pull-requests/)
-* [GitHub Help](https://help.github.com)

data/docs/SECURITY.md

@@ -1,3 +0,0 @@
-# Security Policy
-
-To report a security vulnerability, please email [ben@balter.com](mailto:ben@balter.com).

data/docs/_config.yml

@@ -1,2 +0,0 @@
-title: Gman
-description: A ruby gem to verify if the owner of a given email address, website, or domain is a government agency.

data/gman.gemspec

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-require File.expand_path './lib/gman/version', File.dirname(__FILE__)
-
-Gem::Specification.new do |s|
-  s.name = 'gman'
-  s.summary = <<-SUMMARY
-    Check if a given domain or email address belong to a governemnt entity
-  SUMMARY
-  s.description = <<-DESC
-    A ruby gem to check if the owner of a given email address is working for
-    THE MAN.
-  DESC
-  s.version = Gman::VERSION
-  s.authors = ['Ben Balter']
-  s.email = 'ben.balter@github.com'
-  s.homepage = 'https://github.com/benbalter/gman'
-  s.licenses = ['MIT']
-
-  s.files         = `git ls-files`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map do |f|
-    File.basename(f)
-  end
-
-  s.require_paths = ['lib']
-  s.required_ruby_version = '>= 2.5', '< 4.0'
-
-  s.add_dependency('colored', '~> 1.2')
-  s.add_dependency('iso_country_codes', '~> 0.6')
-  s.add_dependency('naughty_or_nice', '>= 2.1.1')
-  s.add_dependency('public_suffix', '>= 3.0')
-
-  s.add_development_dependency('addressable', '~> 2.3')
-  s.add_development_dependency('mechanize', '~> 2.7')
-  s.add_development_dependency('parallel', '~> 1.6')
-  s.add_development_dependency('pry', '~> 0.10')
-  s.add_development_dependency('rspec', '~> 3.5')
-  s.add_development_dependency('rubocop', '~> 1.0')
-  s.add_development_dependency('rubocop-performance', '~> 1.5')
-  s.add_development_dependency('rubocop-rspec', '~> 2.0')
-  s.add_development_dependency('ruby-prof', '~> 1.4')
-  s.add_development_dependency('ruby-progressbar', '~> 1.10')
-  s.add_development_dependency('swot', '~> 1.0')
-  s.metadata['rubygems_mfa_required'] = 'true'
-end

data/script/add

@@ -1,19 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-#
-# Add one or more domains to a given group, running the standard import checks
-#
-# Usage: script/add [GROUP] [DOMAIN(S)]
-
-require './lib/gman/importer'
-
-if ARGV.length < 2
-  puts 'Usage: script/add [GROUP] [DOMAIN(S)]'
-  exit 1
-end
-
-group = ARGV[0]
-domains = ARGV.drop(1)
-
-Gman.import(group => domains)

data/script/alphabetize

@@ -1,14 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-#
-# Alphabetizes entries in the domains.txt file
-#
-# usage: script/alphabetize
-
-require_relative '../lib/gman'
-require_relative '../lib/gman/importer'
-
-current = Gman::DomainList.current
-current.alphabetize
-current.write

data/script/bootstrap

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-set -ex
-
-bundle install

data/script/cibuild

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-set -ex
-
-bundle exec rspec
-bundle exec rubocop -D -S -a
-bundle exec script/dedupe
-bundle exec gem build gman.gemspec

data/script/console

@@ -1,3 +0,0 @@
-#! /bin/sh
-
-bundle exec pry -r ./lib/gman.rb

data/script/dedupe

@@ -1,22 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'yaml'
-require 'open-uri'
-require './lib/gman'
-require './lib/gman/importer'
-
-current = Gman::DomainList.current
-
-puts 'Checking for duplicate domains in the domain list...'
-puts "Current list contains #{current.count} domains..."
-
-dupe = current.count - current.domains.uniq.count
-puts "Found #{dupe} duplicate domains"
-exit 0 if dupe.zero?
-
-dupes = current.domains.select { |domain| current.domains.count(domain) > 1 }
-
-puts 'Duplicate domains:'
-puts dupes
-exit 1

data/script/profile

@@ -1,21 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'ruby-prof'
-require './lib/gman'
-
-# Pick N random domains directly,
-# without pre-loading the Gman list for an accurate benchmark
-count = (ARGV[0] || 100).to_i
-domains = File.readlines('./config/domains.txt')
-domains = domains.grep(/^[a-z0-9]/i)
-domains = domains.sample(count)
-
-RubyProf.start
-domains.each do |domain|
-  Gman.new("foo@bar.#{domain}").valid?
-end
-
-result = RubyProf.stop
-printer = RubyProf::FlatPrinter.new(result)
-printer.print($stdout)

data/script/prune

@@ -1,22 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Given an array of domains, removes them from the list
-# Example usage: script/prune foo.invalid, bar.invalid, foo.bar.invalid
-
-require_relative '../lib/gman'
-require_relative '../lib/gman/domain_list'
-
-domains = ARGV
-domains = domains.clone.map { |d| d.delete ',' }
-
-list = File.read('./config/domains.txt')
-puts "Starting list: #{Gman::DomainList.current.count} domains"
-
-domains.each do |domain|
-  list.gsub!(/^#{Regexp.escape(domain)}$\n/, '')
-end
-
-File.write './config/domains.txt', list
-
-puts "Ending list: #{Gman::DomainList.current.count} domains"

data/script/reconcile-us

@@ -1,72 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-#
-# Reconciles the USA.gov-maintained list of US domains with domains.txt
-# to show domains listed in the USA.gov-maintained list that we reject and why
-#
-# Usage: script/reconcile-us
-
-require './lib/gman/importer'
-require 'yaml'
-
-ENV['RECONCILING'] = 'true'
-blacklist = ['usagovQUASI']
-source = 'https://raw.githubusercontent.com/GSA/govt-urls/master/government-urls-hierarchical-list.txt'
-
-data = URI.open(source).read
-data = data.split('_' * 74)
-data = data.last.strip
-data = data.split(/\r?\n/).reject(&:empty?)
-
-domains = {}
-group = ''
-data.each do |row|
-  if /^\w/.match?(row)
-    group = row
-    domains[group] = []
-  else
-    domains[group].push row.sub(".\t", '').strip
-  end
-end
-
-domains.reject! { |g, _domain| blacklist.include?(g) }
-importer = Gman::Importer.new(domains)
-
-importer.logger.info "Starting with #{importer.domains.count} domains"
-
-importer.domains.list.each_value do |d|
-  d.map! { |domain| Gman.new(domain).to_s }
-  d.map! { |domain| importer.normalize_domain(domain) }
-end
-
-count = importer.domains.domains.count
-importer.logger.info "Filtered down to #{count} normalized domains"
-
-missing = {}
-importer.domains.list.each do |g, usagovdomains|
-  next unless importer.current.list[g]
-
-  missing[g] = importer.current.list[g] - usagovdomains
-end
-
-missing.reject! { |_key, value| value.empty? }
-
-count = missing.values.count
-importer.logger.info "Found #{count} domains not on the USA.gov list"
-puts "Here's the list of missing domains:"
-puts YAML.dump(missing)
-
-domains = importer.domains.domains
-domains = domains.group_by do |domain|
-  importer.valid_domain?(domain, skip_dupe: true)
-end
-domains.delete(true)
-domains.delete(false)
-domains.delete('locality')
-
-count = domains.values.flatten.count
-importer.logger.info "Calling out #{count} rejected domains"
-
-puts 'Here are the rejected domains and why they were rejected:'
-puts YAML.dump(domains)