mdqt 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 410463d3ecf907b09d8f051c40ade36cfe5b65a7d5fa593f85b4a6f6901ceb0f
-  data.tar.gz: 247dfc548b20015479a6b309940bf04053c09f9d889211a7f78517d2cb2c63d4
+  metadata.gz: 7fbccd299cf9c1a2432b72d55bd74c8c35633529a42137b4ebf82c020a3407d2
+  data.tar.gz: 954092c1deb38ba21374b19bbd00bc2412e92508e95eeda2b7ebb6ada9916ba4
 SHA512:
-  metadata.gz: bdc5aec58160d10cf264d290d6bff4d200124a1bcbb52b377a027d3c1c30e896c7b7ef881d9f972a20935abea94c9f310206eb74ad828b598d42e863627f7708
-  data.tar.gz: e476961884bffe5b9be0438293eba6292246be51a2afc78412f550741cdd97e3eb4a14ca17967bbcac44a1fc0d23f2f96facd080f0d394a4bdaef5868f33ee6c
+  metadata.gz: 064fc97f1e597b7d14febeb892f3176e142bcba1b224bf8e46453d1ead108becb2915788e285adf7f5089c4755a28d1d4a4a4ad92a1e580b2b0e2083042deb89
+  data.tar.gz: 476527f791f3d62efae084ad84632131657383bb7697c6a575f77b12ee6054a1fcbd3c91ec29cd1f304a36f32c67e6522a73281da9d6095f8f9dab2a901b60cb

data/.gitignore

@@ -15,3 +15,4 @@
 /mdq-beta-cert.pem
 /out
 /xout
+out*.xml

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.4.0
+
+### New Features
+- The `check` command will validate XML files against SAML metadata schema and verify signatures
+- A `--validate` switch for `get` forces XML validation, using basic SAML2 metadata schema
+- A `--tls-risky` switch for `get` disables verification of TLS certificates
+
+### Improvements
+- Connection failures now show an explanation (such as TLS problems)
+
+### Fixes
+- "Not Required" was shown when using commands that don't interact with an MDQ server
+
 ## 0.3.1
 
 ### Fixes

data/README.md

@@ -9,6 +9,7 @@ MDQ currently supports:
 
   - Downloading single entities, lists or aggregates
   - Signature verification
+  - Validating metadata against SAML2 schema
   - Saving metadata to disk
   - Caching entity metadata on disk
   - Gzip compression
@@ -33,7 +34,7 @@ To install system-wide on your default Ruby, use
 
     $ sudo gem install mdqt
 
-If using a per-user Ruby via `rbenv` or similar, you'll just need
+If using a per-user Ruby via `rbenv` or similar, you'll need
 
     $ gem install mdqt
 
@@ -49,6 +50,11 @@ and then execute:
 
     $ bundle
 
+### As a Docker container
+
+(Experimental)
+See the instructions at [MDQT-Container](https://github.com/Digital-Identity-Labs/mdqt-container)
+
 ### Extra steps for verifying signed metadata
 
 MDQT can check that metadata has not been tampered with by verifying its
@@ -138,6 +144,22 @@ It's possible to pass more than one certificate by separating them with commas
 
     $ mdqt get --verify-with myfederation.pem,previous.pem https://indiid.net/idp/shibboleth
 
+Basic XML correctness and validation against SAML2 Metadata schema can be enabled with the
+`--validate` switch:
+
+    $ mdqt get --validate https://indiid.net/idp/shibboleth
+
+If you need to check metadata that has already been downloaded then try the `check`
+command:
+
+    $ mdqt check metadata.xml # Just validate
+    $ mdqt check --verify-with myfederation.pem metadata.xml # Verify signature too
+
+You shouldn't need to *validate* XML from a trusted MDQ service such as one run by a
+national federation. You should however always *verify* the signature of XML sent over an unencrypyted HTTP connection,
+and probably even over HTTPS. MDQT's validation check is mostly for use when writing
+or debugging your own MDQ service.
+
 ### Saving metadata as files
 
 The simplest way to save metadata is to redirect output from the `get` command:
@@ -163,7 +185,7 @@ For more information about current settings, download results, and so on, add
 
     $mdqt get --verbose http://entity.ac.uk/shibboleth
 
-To convert normal URI entity IDs into MDQ SHA1 hashed transformed identifiers just use the `transform` command:
+To convert normal URI entity IDs into MDQ SHA1 hashed transformed identifiers use the `transform` command:
 
     $ mdqt transform http://example.org/service
 

data/exe/mdqt

@@ -30,9 +30,11 @@ Commander.configure do
     c.option '--service URL', String, 'MDQ service to search for entities. Defaults to MDQT_SERVICE or MDQ_BASE_URL env variables'
     c.option '--cache', "Cache downloads and try to fetch from cache where appropriate"
     c.option '--verify-with PATHS', Array, 'Validate downloads using specified certificates'
+    c.option '--validate', 'Validate downloaded metadata against SAML2 schema (not normally needed)'
     #c.option '--stdin', 'accept one or more entity ids from STDIN'
     c.option '--all', 'Request all entity records'
     c.option '--explain', 'Show details of client request and server response'
+    c.option '--tls-risky', "Don't check certificate used for TLS (usually a bad idea)"
     c.option '--save-to PATH', String, 'Write all data to files in the specified directory'
     c.option '--link-id', 'If saving files, save files with aliases (requires `--save-to`)'
     c.action do |args, options|
@@ -62,5 +64,16 @@ Commander.configure do
     end
   end
 
+  command :check do |c|
+    c.syntax = 'mdqt check XML_FILENAME CERTIFICATE_FILENAME'
+    c.description = 'Validate XML and check signatures'
+    c.option '--verify-with PATHS', Array, 'Validate file using specified certificates'
+    c.action do |args, options|
+      options.default MDQT::CLI::Defaults.cli_defaults
+      options.default({service: :not_required, validate: true })
+      MDQT::CLI::Check.run(args, options)
+    end
+  end
+
 end
 

data/lib/mdqt/cli.rb

@@ -9,6 +9,7 @@ module MDQT
     require 'mdqt/cli/reset'
     require 'mdqt/cli/version'
     require 'mdqt/cli/transform'
+    require 'mdqt/cli/check'
 
 
   end

data/lib/mdqt/cli/base.rb

@@ -4,6 +4,7 @@ module MDQT
 
     class Base
 
+      require 'mdqt/cli'
       require 'pastel'
 
       def self.run(args, options)
@@ -34,11 +35,14 @@ module MDQT
       def self.introduce(args, options)
         if options.verbose
           STDERR.puts "MDQT version #{MDQT::VERSION}"
-          STDERR.puts "Using #{options.service}"
+          STDERR.puts "Using #{options.service}" unless options.service == :not_required
           STDERR.puts "Caching is #{options.cache ? 'on' : 'off'}"
+          STDERR.print "XML validation is #{MDQT::Client.verification_available? ? 'available' : 'not available'}"
+          STDERR.puts  " #{options.validate ? "and active" : "but inactive"} for this request" if MDQT::Client.verification_available?
           STDERR.print "Signature verification is #{MDQT::Client.verification_available? ? 'available' : 'not available'}"
           STDERR.puts  " #{options.verify_with ? "and active" : "but inactive"} for this request" if MDQT::Client.verification_available?
           STDERR.puts "Output directory for saved files is: #{File.absolute_path(options.save_to)}" if options.save_to
+          STDERR.puts("Warning! TLS certificate verification has been disabled!") if options.tls_risky
           STDERR.puts
         end
       end
@@ -104,7 +108,7 @@ module MDQT
       end
 
       def advise_on_xml_signing_support
-        hey "XML signature validation is not available. Install the 'xmldsig' gem if you can." unless MDQT::Client.verification_available?
+        hey "XML signature verification and XML validation are not available. Install the 'xmldsig' gem if you can." unless MDQT::Client.verification_available?
       end
 
       def extract_certificate_paths(cert_paths = options.verify_with)

data/lib/mdqt/cli/check.rb

@@ -0,0 +1,82 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Check < Base
+
+      def run
+
+        options.validate = true
+
+        advise_on_xml_signing_support
+        halt!("Cannot check a metadata file without XML support: please install additional gems") unless MDQT::Client.verification_available?
+
+        client = MDQT::Client.new(
+            options.service,
+            verbose: options.verbose,
+            explain: options.explain ? true : false,
+            )
+
+        cert_paths = options.verify_with ? extract_certificate_paths(options.verify_with) : []
+
+        args.each do |filename|
+
+          filename = File.absolute_path(filename)
+          file = client.open_metadata(filename)
+
+          halt!("Cannot access file #{filename}") unless file.readable?
+
+          halt!("XML validation failed for #{filename}:\n#{file.validation_error}") unless file.valid?
+          btw"File #{filename} is valid SAML Metadata XML"
+
+
+          if options.verify_with
+            halt! "XML in #{filename} is not signed, cannot verify!" unless file.signed?
+            halt! "The signed XML for #{filename} cannot be verified using #{cert_paths.to_sentence}" unless file.verified_signature?(cert_paths)
+            btw "Signed XML for #{filename} has been verified using '#{cert_paths.to_sentence}'"
+          end
+
+          yay "#{filename} OK"
+        end
+
+
+      end
+
+
+      def verify_results(results)
+
+        # if options.validate
+        #   results.each do |result|
+        #     next unless result.ok?
+        #     halt! "The data for #{result.identifier} is not valid when checked against schema:\n#{result.validation_error}" unless result.valid?
+        #     btw "Data for #{result.identifier.empty? ? 'aggregate' : result.identifier } has been validated against schema" ## FIXME - needs constistent #label maybe?
+        #   end
+        # end
+        #
+        # return results unless options.verify_with
+        #
+        # cert_paths = extract_certificate_paths(options.verify_with)
+        #
+        # results.each do |result|
+        #   next unless result.ok?
+        #   halt! "Data from #{options.service} is not signed, cannot verify!" unless result.signed?
+        #   halt! "The data for #{result.identifier} cannot be verified using #{cert_paths.to_sentence}" unless result.verified_signature?(cert_paths)
+        #   btw "Data for #{result.identifier.empty? ? 'aggregate' : result.identifier } has been verified using '#{cert_paths.to_sentence}'" ## FIXME - needs constistent #label maybe?
+        # end
+        #
+        # results
+
+      end
+
+    end
+
+    private
+
+
+  end
+
+end
+
+

data/lib/mdqt/cli/get.rb

@@ -26,6 +26,7 @@ module MDQT
             options.service,
             verbose: options.verbose,
             explain: options.explain ? true : false,
+            tls_risky: options.tls_risky ? true : false,
             cache_type: options.cache ? :file : :none,
         )
 
@@ -35,6 +36,14 @@ module MDQT
 
       def verify_results(results)
 
+        if options.validate
+          results.each do |result|
+            next unless result.ok?
+            halt! "The data for #{result.identifier} is not valid when checked against schema:\n#{result.validation_error}" unless result.valid?
+            btw "Data for #{result.identifier.empty? ? 'aggregate' : result.identifier } has been validated against schema" ## FIXME - needs constistent #label maybe?
+          end
+        end
+
         return results unless options.verify_with
 
         cert_paths = extract_certificate_paths(options.verify_with)

data/lib/mdqt/client.rb

@@ -4,6 +4,7 @@ module MDQT
     require 'rubygems'
     require 'mdqt/client/metadata_service'
     require 'mdqt/client/metadata_validator'
+    require 'mdqt/client/metadata_file'
     require 'mdqt/client/identifier_utils'
 
     begin
@@ -20,19 +21,22 @@ module MDQT
 
     def initialize(base_url, options={})
 
-      @base_url   = base_url
-      @verbose    = options[:verbose] || false
-      @explain    = options[:explain] || false
+      @base_url        = base_url
+      @verbose         = options[:verbose] || false
+      @explain         = options[:explain] || false
+      @tls_cert_check  = options[:tls_risky] ? false : true
       @cache_type = options[:cache_type] || :none
 
-      @md_service = MetadataService.new(@base_url, verbose: @verbose, cache_type: @cache_type, explain: @explain)
+      @md_service = MetadataService.new(@base_url, verbose: @verbose, cache_type: @cache_type, explain: @explain, tls_cert_check: tls_cert_check?)
 
     end
 
-    def get_metadata(entity_id)
+    def open_metadata(filename)
+      MetadataFile.new(filename, verbose: @verbose)
+    end
 
+    def get_metadata(entity_id)
       md_service.get(entity_id)
-
     end
 
     def transform_uri(uri)
@@ -51,6 +55,10 @@ module MDQT
       @explain
     end
 
+    def tls_cert_check?
+      @tls_cert_check
+    end
+
     def cache_type
       @cache_type
     end

data/lib/mdqt/client/metadata_file.rb

@@ -0,0 +1,89 @@
+module MDQT
+  class Client
+
+    class MetadataFile
+
+      require 'digest'
+
+      def initialize(filename, options = {})
+        @filename = filename
+        @identifier = nil
+        @data = nil
+        @expires = nil
+        @etag = nil
+        @last_modified = nil
+        @explanation = {}
+      end
+
+      def filename
+        @filename
+      end
+
+      def identifier
+        @identifier
+      end
+
+      def data
+        @data ||= File.read(filename)
+      end
+
+      def readable?
+        File.readable?(filename)
+      end
+
+      def type
+        @type
+      end
+
+      def expires
+        @expires
+      end
+
+      def etag
+        @etag
+      end
+
+      def last_modified
+        @last_modified
+      end
+
+      def ok?
+        @ok
+      end
+
+      def signed?
+        @data.include? "Signature" # This is... not great
+      end
+
+      def verified_signature?(certs = [], _ = {})
+        validator = MetadataValidator.new(certs: [certs].flatten)
+        validator.verified_signature?(self)
+      end
+
+      def valid?
+        validator = MetadataValidator.new
+        validator.valid?(self)
+      end
+
+      def validation_error
+        validator = MetadataValidator.new
+        validator.validation_error(self)
+      end
+
+      def canonical_filename
+        if identifier.empty?
+          @filename = "aggregate-#{Digest::SHA1.hexdigest(@service)}.xml"
+        else
+          @filename ||= identifier.start_with?("{sha1}") ?
+                            "#{@identifier.gsub("{sha1}","")}.xml" :
+                            "#{Digest::SHA1.hexdigest(@identifier)}.xml"
+        end
+      end
+
+      private
+
+    end
+
+  end
+
+end

data/lib/mdqt/client/metadata_response.rb

@@ -74,11 +74,21 @@ module MDQT
       end
 
       def verified_signature?(certs = [], _ = {})
-        return true unless ok?
+        return true unless ok?  # CHECK ?
         validator = MetadataValidator.new(certs: [certs].flatten)
         validator.verified_signature?(self)
       end
 
+      def valid?
+        validator = MetadataValidator.new
+        validator.valid?(self)
+      end
+
+      def validation_error
+        validator = MetadataValidator.new
+        validator.validation_error(self)
+      end
+
       def filename
         if identifier.empty?
           @filename = "aggregate-#{Digest::SHA1.hexdigest(@service)}.xml"
@@ -123,6 +133,9 @@ module MDQT
         @explanation
       end
 
+      private
+
+
     end
 
   end

data/lib/mdqt/client/metadata_service.rb

@@ -24,7 +24,7 @@ module MDQT
         @store_config = options[:cache_store]
         @verbose = options[:verbose] ? true : false
         @explain = options[:explain] ? true : false
-
+        @tls_cert_check  = options[:tls_cert_check] ? true : false
       end
 
       def base_url
@@ -42,7 +42,7 @@ module MDQT
             req.options.open_timeout = 5
           end
         rescue Faraday::ConnectionFailed => oops
-          abort "Error - can't connect to MDQ service at URL #{base_url}"
+          abort "Error - can't connect to MDQ service at URL #{base_url}: #{oops.to_s}"
         end
 
         MetadataResponse.new(entity_id, base_url, http_response, explain: explain?)
@@ -70,6 +70,10 @@ module MDQT
         @explain
       end
 
+      def tls_cert_check?
+        @tls_cert_check
+      end
+
       def cache?
         cache_type == :none ? false : true
       end
@@ -116,6 +120,7 @@ module MDQT
           faraday.use FaradayMiddleware::Gzip
           faraday.use FaradayMiddleware::FollowRedirects
           faraday.use :http_cache, faraday_cache_config if cache?
+          faraday.ssl.verify = tls_cert_check?
           faraday.headers['Accept']         = 'application/samlmetadata+xml'
           faraday.headers['Accept-Charset'] = 'utf-8'
           faraday.headers['User-Agent']     = "MDQT v#{MDQT::VERSION}"