mdqt 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7fbccd299cf9c1a2432b72d55bd74c8c35633529a42137b4ebf82c020a3407d2
-  data.tar.gz: 954092c1deb38ba21374b19bbd00bc2412e92508e95eeda2b7ebb6ada9916ba4
+  metadata.gz: c8731e9b40b397c638c8bb0e11acc9c954a8aaf940cd558683ff0e14df68a668
+  data.tar.gz: a009f1591877bb571ab8d1f967545dbc836b7f3953e267548433352a122bc449
 SHA512:
-  metadata.gz: 064fc97f1e597b7d14febeb892f3176e142bcba1b224bf8e46453d1ead108becb2915788e285adf7f5089c4755a28d1d4a4a4ad92a1e580b2b0e2083042deb89
-  data.tar.gz: 476527f791f3d62efae084ad84632131657383bb7697c6a575f77b12ee6054a1fcbd3c91ec29cd1f304a36f32c67e6522a73281da9d6095f8f9dab2a901b60cb
+  metadata.gz: e802b9620268525cf5b4249c59e604ba5345e45805b15e9c57ed4daee38271712f4d8f9bc8213a2883c9ac39f8a303bbeaec1dd8eacc09f223a61aa4d6ae5b07
+  data.tar.gz: b37ed2ddf5e7a9212e4ab5f722a77b8c2fe0bde7e6e10c74203f7fe2b7c9863f873eb491e5fea0d121b2281e6955122d9f0d21c3d9d201c4c82c8743aa952067

data/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '17 4 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'ruby' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

data/.github/workflows/ruby.yml

@@ -0,0 +1,35 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.6', '2.7', '3.0']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: bundle exec rake

data/.gitignore

@@ -16,3 +16,10 @@
 /out
 /xout
 out*.xml
+/.idea
+/*.xml
+/bt
+*.html
+/t/aggregate.xml
+/t/indiid.xml
+/t/uom.xml

data/.ruby-version

@@ -1 +1 @@
-2.5.0
+2.6.6

data/.tool-versions

@@ -0,0 +1 @@
+ruby 3.0.2

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## 0.5.0
+
+### New Features
+
+- New `entities` command extracts entity IDs and sha1 hashes from metadata files on disk
+- New `ln` command will create symlinks to files using their sha1 hashes
+- New `ls` command will list the entity IDs of metadata files
+- New `list` command lists all entity IDs available from the MDQ service
+- New `services` command shows known MDQ services and aliases
+- New `rename` command renames metadata files to use their sha1 hash as a name 
+- New `url` command shows the full url for an entity at the MDQ service
+
+### Improvements
+
+- Known MDQ services can be specified using simple aliases as well as URLs
+- Caching is now on by default
+- `--refresh` options forces downloads and ignores cached data
+- Cache is cleaned whenever `get` is used, to remove expired files
+- Added default service details for DFN
+- Tidier output when stopped with ctrl-c
+
+### Fixes
+
+- Compatible with Ruby 3
+- Updated dependencies to latest versions
+- Improved test reliability and added more tests
+- Extended timeouts to better handle slow networks
+
 ## 0.4.0
 
 ### New Features

data/Gemfile

@@ -5,4 +5,6 @@ gemspec
 
 gem 'simplecov', require: false, group: :test
 
-gem 'xmldsig'
+gem 'pry'
+gem 'nokogiri', '~> 1.12', '>= 1.12.5'
+gem 'xmldsig', '>= 0.6.6'

data/README.md

@@ -5,14 +5,21 @@
 MDQT is small library and commandline tool to query MDQ services for SAML metadata.
 You could do this with `curl` and `xmlsec1` but it's a little more convenient to use `mdqt` instead.
 
-MDQ currently supports:
+MDQT also has features for managing local metadata files, to help when running an MDQ service or a Shibboleth IdP or SP.
+
+MDQ currently provides these features:
 
   - Downloading single entities, lists or aggregates
   - Signature verification
   - Validating metadata against SAML2 schema
   - Saving metadata to disk
-  - Caching entity metadata on disk
-  - Gzip compression
+  - Extracting entity IDs from both aggregate and individual metadata files
+  - Renaming metadata files to their entity ID sha1 hashes
+  - Creating sha1 hash symlinks to metadata files
+  - Listing the entity IDs of downloaded metadata files
+  - Showing the full URL of an entity
+  - Caching entity metadata and using Gzip compression
+
 
 ## MDQ?
 
@@ -34,7 +41,7 @@ To install system-wide on your default Ruby, use
 
     $ sudo gem install mdqt
 
-If using a per-user Ruby via `rbenv` or similar, you'll need
+If using a per-user Ruby via `rbenv`, `asdf` or similar, you'll need
 
     $ gem install mdqt
 
@@ -63,7 +70,7 @@ signature. Some MDQ services use unencrypted HTTP connections and rely
 
 MDQT supports signature verification but requires a Ruby library called
 Nokogiri to do the hard work. Nokogiri is fast and useful but can sometimes
-be awkward to install for non-developers (it requires a C development
+be awkward to install for non-developers (it can sometimes require a C development
 environment and various XML libraries). To make it easier to install a basic MDQT we've made
 XML signature verification an optional feature.
 
@@ -102,6 +109,12 @@ service. Set `MDQT_SERVICE` or `MDQ_BASE_URL` to the base URL of your MDQ servic
 Finally, if you don't specify an MDQ service with `--service` or `MDQT_SERVICE` then `mdqt` *might* be
 able to guess your local NREN's MDQ service. Do not do this in production!
 
+If an MDQ service is known to MDQT it can be selected using an alias:
+
+    $ mdqt get --service incommon  http://entity.edu/shibboleth
+
+You can see known services and their aliases using `mdqt services`
+
 ### Downloading entity metadata
 
 Downloading entity metadata to STDOUT:
@@ -114,6 +127,8 @@ Using the sha1 hashed version of entity IDs requires quotes or escaping in some
 
     $ mdqt get \{sha1\}52e2065fc0d53744e8d4ee2c2f30696ebfc5def9
 
+    $ mdqt get [sha1]52e2065fc0d53744e8d4ee2c2f30696ebfc5def9
+
 Requesting all metadata from an MDQ endpoint is done by specifying `--all`:
 
     $ mdqt get --all
@@ -127,6 +142,10 @@ directory.
 
     $ mdqt get --cache --service https://mdq.example.com/mdq http://entity.ac.uk/shibboleth
 
+Caching is now on by default. To force a single command to *not* use the cache, include `--reset`
+
+    $ mdqt get --reset --service https://mdq.example.com/mdq http://entity.ac.uk/shibboleth
+
 You can clear the cache by using the `reset` command:
 
     $ mdqt reset
@@ -170,7 +189,7 @@ MDQT also offers the `--save-to` option to write all metadata into a directory
 
     $ mdqt get http://entity.ac.uk/shibboleth --save-to metadata_directory
 
-The  `--save-to` option requires a directory to be specified. All files will be saved
+The `--save-to` option requires a directory to be specified. All files will be saved
 with a name based on their transformed identifier (sha1 hash) such as
 `77603e0cbda1e00d50373ca8ca20a375f5d1f171.xml`
 
@@ -178,10 +197,10 @@ By adding the `--link-id' flag alternative filenames will be linked to the
 original file (this is currently a little experimental) to make it easier
 to look up the correct file using other identifiers.
 
-### Other features
+### Other Features
 
 For more information about current settings, download results, and so on, add
-`--verbose` to commands.
+`--verbose` to commands:
 
     $mdqt get --verbose http://entity.ac.uk/shibboleth
 
@@ -195,11 +214,42 @@ To see more details of what is being sent and received by a `get` command add th
 
     $ mdqt get --explain --service https://mdq.example.com/mdq  http://entity.ac.uk/shibboleth
 
-MDQT will then show a table of sent and recieved headers which may be useful when debugging servers.
+MDQT will then show a table of sent and received headers which may be useful when debugging servers.
+
+To extract a list of all entity IDs from a file:
+
+    $ mdqt entities metadata.xml
+
+    $ mdqt entities --sha1 metadata.xml
+
+To create sha1 symlinks to a metadata file:
+
+    $ mdqt ln example_idp.xml
+
+To rename a file to its entity ID sha1 has:
+
+    $ mdqt rename example_idp.xml
+
+To list the entity IDs of files in a directory:
+
+    $ mdqt ls
+
+To list all entities available at an MDQ service:
+
+    $ mdqt list
+
+To show the MDQ services known to MDQT, and their aliases:
+
+    $ mdqt services
+
+To show the full MDQ URL of an entity
+
+    $ mdqt url http://entity.ac.uk/shibboleth
+
 
 ## Library Usage
 
-Please don't! This gem is very early in development and the API is not stable. Later
+Please don't! This gem is early in development and the API is not stable. Later
 releases of this gem will provide a simple library to use in other Ruby applications.
 
 ## Development

data/cucumber.yml

@@ -0,0 +1,2 @@
+default: --publish
+

data/exe/mdqt

@@ -5,12 +5,17 @@ require 'mdqt/version'
 
 require 'commander'
 
+Signal.trap('SIGINT') do
+  puts 'Received signal, halting'
+  exit 1
+end
+
 Commander.configure do
   program :name, 'mdqt'
   program :version, MDQT::VERSION
   program :description, 'MDQ SAML metadata client'
 
-  global_option '--verbose'
+  #global_option '--verbose'
 
   default_command :help
 
@@ -19,7 +24,7 @@ Commander.configure do
     c.description = 'Show version of MDQT'
     c.action do |args, options|
       options.default MDQT::CLI::Defaults.cli_defaults
-      options.default({service: :not_required })
+      options.default({ service: :not_required })
       MDQT::CLI::Version.run(args, options)
     end
   end
@@ -28,7 +33,8 @@ Commander.configure do
     c.syntax = 'mdqt get [options] entityidentifier '
     c.description = 'Download one entity record or an aggregate of entity records'
     c.option '--service URL', String, 'MDQ service to search for entities. Defaults to MDQT_SERVICE or MDQ_BASE_URL env variables'
-    c.option '--cache', "Cache downloads and try to fetch from cache where appropriate"
+    c.option '--cache', "Cache downloads and try to fetch from cache where appropriate (deprecated)"
+    c.option '--refresh', "Never cache (will prevent --cache)"
     c.option '--verify-with PATHS', Array, 'Validate downloads using specified certificates'
     c.option '--validate', 'Validate downloaded metadata against SAML2 schema (not normally needed)'
     #c.option '--stdin', 'accept one or more entity ids from STDIN'
@@ -37,9 +43,10 @@ Commander.configure do
     c.option '--tls-risky', "Don't check certificate used for TLS (usually a bad idea)"
     c.option '--save-to PATH', String, 'Write all data to files in the specified directory'
     c.option '--link-id', 'If saving files, save files with aliases (requires `--save-to`)'
+    c.option '--verbose', 'Display extra information on stderr'
     c.action do |args, options|
       options.default MDQT::CLI::Defaults.cli_defaults
-      options.default({service: MDQT::CLI::Defaults.base_url }) if options.service.nil?
+      options.default({ service: MDQT::CLI::Defaults.base_url }) if options.service.to_s == ''
       MDQT::CLI::Get.run(args, options)
     end
   end
@@ -47,9 +54,10 @@ Commander.configure do
   command :reset do |c|
     c.syntax = 'mdqt reset'
     c.description = 'Delete all cached data'
+    c.option '--verbose', 'Display extra information on stderr'
     c.action do |args, options|
       options.default MDQT::CLI::Defaults.cli_defaults
-      options.default({service: :not_required })
+      options.default({ service: :not_required })
       MDQT::CLI::Reset.run(args, options)
     end
   end
@@ -57,9 +65,10 @@ Commander.configure do
   command :transform do |c|
     c.syntax = 'mdqt transform ENTITYIDS'
     c.description = 'Show transformed entity IDs'
+    c.option '--verbose', 'Display extra information on stderr'
     c.action do |args, options|
       options.default MDQT::CLI::Defaults.cli_defaults
-      options.default({service: :not_required })
+      options.default({ service: :not_required })
       MDQT::CLI::Transform.run(args, options)
     end
   end
@@ -67,13 +76,101 @@ Commander.configure do
   command :check do |c|
     c.syntax = 'mdqt check XML_FILENAME CERTIFICATE_FILENAME'
     c.description = 'Validate XML and check signatures'
+    c.option '--verbose', 'Display extra information on stderr'
     c.option '--verify-with PATHS', Array, 'Validate file using specified certificates'
     c.action do |args, options|
       options.default MDQT::CLI::Defaults.cli_defaults
-      options.default({service: :not_required, validate: true })
+      options.default({ service: :not_required, validate: true })
       MDQT::CLI::Check.run(args, options)
     end
   end
 
+  command :entities do |c|
+    c.syntax = 'mdqt entities XML_FILENAME'
+    c.description = 'Extract entity IDs from a metadata file'
+    c.option '--sha1', 'include the sha1 hash for each entity ID'
+    c.action do |args, options|
+      args = Dir.glob("*.xml") unless args && !args.empty?
+      options.default MDQT::CLI::Defaults.cli_defaults
+      options.default({ service: :not_required })
+      MDQT::CLI::Entities.run(args, options)
+    end
+  end
+
+  command :ln do |c|
+    c.syntax = 'mdqt ln XML_FILENAME'
+    c.description = 'Create a soft link to the file using an sha1 hash of the entityID'
+    c.option '--force', 'Overwrite any existing links'
+    c.option '--verbose', 'Display extra information on stderr'
+    c.action do |args, options|
+      args = Dir.glob("*.xml") unless args && !args.empty?
+      options.default MDQT::CLI::Defaults.cli_defaults
+      options.default({ service: :not_required })
+      MDQT::CLI::Ln.run(args, options)
+    end
+  end
+
+  command :ls do |c|
+    c.syntax = 'mdqt ls XML_FILENAME/DIRECTORY'
+    c.description = 'List valid metadata files in directory'
+    c.option '--verbose', 'Display extra information on stderr'
+    c.action do |args, options|
+      args = Dir.glob("*.xml") unless args && !args.empty?
+      options.default MDQT::CLI::Defaults.cli_defaults
+      options.default({ service: :not_required })
+      MDQT::CLI::Ls.run(args, options)
+    end
+  end
+
+  command :list do |c|
+    c.syntax = 'mdqt list [options]'
+    c.description = 'List entities available at the MDQ service'
+    c.option '--service URL', String, 'MDQ service to search for entities. Defaults to MDQT_SERVICE or MDQ_BASE_URL env variables'
+    c.option '--cache', "Cache downloads and try to fetch from cache where appropriate (deprecated)"
+    c.option '--refresh', "Never cache (will prevent --cache)"
+    #c.option '--stdin', 'accept one or more entity ids from STDIN'
+    c.option '--verbose', 'Display extra information on stderr'
+    c.action do |args, options|
+      options.default MDQT::CLI::Defaults.cli_defaults
+      options.default({ service: MDQT::CLI::Defaults.base_url }) if options.service.to_s == ''
+      MDQT::CLI::List.run(args, options)
+    end
+  end
+
+  command :services do |c|
+    c.syntax = 'mdqt services'
+    c.description = 'List URLs and aliases for known MDQ services'
+    c.action do |args, options|
+      options.default MDQT::CLI::Defaults.cli_defaults
+      options.default({ service: :not_required })
+      MDQT::CLI::Services.run(args, options)
+    end
+  end
+
+  command :rename do |c|
+    c.syntax = 'mdqt rename XML_FILENAME'
+    c.description = 'Rename a file using the sha1 hash of its entityID'
+    c.option '--force', 'Overwrite any existing files with that name'
+    c.option '--verbose', 'Display extra information on stderr'
+    c.action do |args, options|
+      args = Dir.glob("*.xml") unless args && !args.empty?
+      options.default MDQT::CLI::Defaults.cli_defaults
+      options.default({ service: :not_required })
+      MDQT::CLI::Rename.run(args, options)
+    end
+  end
+
+  command :url do |c|
+    c.syntax = 'mdqt url ENTITYIDS'
+    c.description = 'List URLs for each entity ID at the MDQ service'
+    c.option '--verbose', 'Display extra information on stderr'
+    c.option '--service URL', String, 'MDQ service to search for entities. Defaults to MDQT_SERVICE or MDQ_BASE_URL env variables'
+    c.action do |args, options|
+      options.default MDQT::CLI::Defaults.cli_defaults
+      options.default({ service: MDQT::CLI::Defaults.base_url }) if options.service.to_s == ''
+      MDQT::CLI::URL.run(args, options)
+    end
+  end
+
 end
 

data/lib/mdqt/cli/base.rb

@@ -18,7 +18,11 @@ module MDQT
       end
 
       def self.check_requirements(args, options)
-        abort "Error: No MDQ service URL has been specified." unless options.service
+        
+        unless options.service == :not_required
+          abort "No MDQ service URL has been specified. Please use --service, MDQT_SERVICE or MDQ_BASE_URL" unless service_url(options).to_s.start_with?("http")
+        end
+
         if options.save_to
           dir = options.save_to
           begin
@@ -35,8 +39,8 @@ module MDQT
       def self.introduce(args, options)
         if options.verbose
           STDERR.puts "MDQT version #{MDQT::VERSION}"
-          STDERR.puts "Using #{options.service}" unless options.service == :not_required
-          STDERR.puts "Caching is #{options.cache ? 'on' : 'off'}"
+          STDERR.puts "Using #{service_url(options)}" unless options.service == :not_required
+          STDERR.puts "Caching is #{MDQT::CLI::CacheControl.caching_on?(options) ? 'on' : 'off'}"
           STDERR.print "XML validation is #{MDQT::Client.verification_available? ? 'available' : 'not available'}"
           STDERR.puts  " #{options.validate ? "and active" : "but inactive"} for this request" if MDQT::Client.verification_available?
           STDERR.print "Signature verification is #{MDQT::Client.verification_available? ? 'available' : 'not available'}"
@@ -81,6 +85,24 @@ module MDQT
         @options
       end
 
+      def self.service_url(options)
+
+        return nil if options.service == :not_required
+
+        choice = options.service.to_s.strip
+
+        if choice.downcase.start_with? "http"
+          choice
+        else
+          Defaults.lookup_service_alias(choice)
+        end
+
+      end
+
+      def service_url(options)
+        self.class.service_url(options)
+      end
+
       def output(response)
         if response.ok?
           yay response.message
@@ -131,6 +153,10 @@ module MDQT
         @pastel ||= Pastel.new
       end
 
+      def say(text)
+        STDOUT.puts(text)
+      end
+
       def hey(comment)
         STDERR.puts(comment)
       end
@@ -146,6 +172,7 @@ module MDQT
       def halt!(comment)
         abort pastel.red("Error: #{comment}")
       end
+
       def run
         halt! "No action has been defined for this command!"
       end

data/lib/mdqt/cli/cache_control.rb

@@ -0,0 +1,25 @@
+module MDQT
+  class CLI
+
+    class CacheControl
+
+      class << self
+
+        def caching_on?(options)
+          return false if cache_type(options) == :none
+          true
+        end
+
+        def cache_type(options)
+          return :none if options.refresh
+          return :memcache if options.cache && options.memcache
+          return :file if options.cache
+          :none
+        end
+
+      end
+
+    end
+  end
+
+end

data/lib/mdqt/cli/check.rb

@@ -14,10 +14,10 @@ module MDQT
         halt!("Cannot check a metadata file without XML support: please install additional gems") unless MDQT::Client.verification_available?
 
         client = MDQT::Client.new(
-            options.service,
-            verbose: options.verbose,
-            explain: options.explain ? true : false,
-            )
+          service_url(options),
+          verbose: options.verbose,
+          explain: options.explain ? true : false,
+        )
 
         cert_paths = options.verify_with ? extract_certificate_paths(options.verify_with) : []
 
@@ -29,8 +29,7 @@ module MDQT
           halt!("Cannot access file #{filename}") unless file.readable?
 
           halt!("XML validation failed for #{filename}:\n#{file.validation_error}") unless file.valid?
-          btw"File #{filename} is valid SAML Metadata XML"
-
+          btw "File #{filename} is valid SAML Metadata XML"
 
           if options.verify_with
             halt! "XML in #{filename} is not signed, cannot verify!" unless file.signed?
@@ -41,10 +40,8 @@ module MDQT
           yay "#{filename} OK"
         end
 
-
       end
 
-
       def verify_results(results)
 
         # if options.validate
@@ -74,7 +71,6 @@ module MDQT
 
     private
 
-
   end
 
 end