mdqt 0.4.0 → 0.5.0

data/lib/mdqt/cli/defaults.rb

@@ -6,7 +6,9 @@ module MDQT
       class << self
 
         def base_url
+
           ENV['MDQT_SERVICE'] || ENV['MDQ_BASE_URL'] || guess_service
+
         end
 
         def force_hash?
@@ -16,7 +18,8 @@ module MDQT
         def cli_defaults
           {
             hash: force_hash?,
-            cache: false
+            cache: true,
+            refresh: false
           }
         end
 
@@ -24,21 +27,39 @@ module MDQT
 
           locale = ENV['LANG']
 
-          #STDERR.puts("Detected locale #{locale}")
+          service = services.find { |s| s[:locale] == locale }
+          #service ||= services.first
 
-          service = case locale
-                    when 'en_GB.UTF-8'
-                      'http://mdq.ukfederation.org.uk/'
-                    when 'en_US.UTF-8'
-                      'http://mdq-beta.incommon.org/global'
-                    else
-                      abort "Please specify an MDQ service using --service, MDQT_SERVICE or MDQ_BASE_URL"
-                    end
+          if service
+            url = service[:url]
+            STDERR.puts "MDQT is assuming that you want to use #{url}\nPlease configure this using --service, MDQT_SERVICE or MDQ_BASE_URL\n\n"
+            url
+          else
+            nil
+          end
 
-          STDERR.puts "MDQT is assuming that you want to use #{service}\nPlease configure this using --service, MDQT_SERVICE or MDQ_BASE_URL\n\n"
+        end
 
-          service
+        def lookup_service_alias(srv_alias)
+          service = services.find { |s| s[:alias].to_s.downcase.to_sym == srv_alias.to_s.downcase.to_sym }
+          service ? service[:url] : nil
+        end
 
+        def services
+          [
+            { alias: "ukamf",
+              locale: "en_GB.UTF-8",
+              url: "http://mdq.ukfederation.org.uk/"
+            },
+            { alias: "incommon",
+              locale: "en_US.UTF-8",
+              url: "https://mdq.incommon.org/"
+            },
+            { alias: "dfn",
+              locale: "de_utf8",
+              url: "https://mdq.aai.dfn.de"
+            },
+          ]
         end
 
       end

data/lib/mdqt/cli/entities.rb

@@ -0,0 +1,47 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Entities < Base
+
+      def run
+
+        options.validate = true
+
+        advise_on_xml_signing_support
+        halt!("Cannot check a metadata file without XML support: please install additional gems") unless MDQT::Client.verification_available?
+
+        client = MDQT::Client.new(
+          service_url(options),
+          verbose: options.verbose,
+          explain: options.explain ? true : false,
+        )
+
+        args.each do |filename|
+
+          file = client.open_metadata(filename)
+
+          halt!("Cannot access file #{filename}") unless file.readable?
+
+          halt!("XML validation failed for #{filename}:\n#{file.validation_error}") unless file.valid?
+
+          file.entity_ids.each do |id|
+            id = options.sha1 ? [id, MDQT::Client::IdentifierUtils.transform_uri(id)].join(" ") : id
+            say(id)
+          end
+
+        end
+
+      end
+
+    end
+
+    private
+
+  end
+
+end
+
+

data/lib/mdqt/cli/get.rb

@@ -23,14 +23,14 @@ module MDQT
       def get_responses
 
         client = MDQT::Client.new(
-            options.service,
-            verbose: options.verbose,
-            explain: options.explain ? true : false,
-            tls_risky: options.tls_risky ? true : false,
-            cache_type: options.cache ? :file : :none,
+          service_url(options),
+          verbose: options.verbose,
+          explain: options.explain ? true : false,
+          tls_risky: options.tls_risky ? true : false,
+          cache_type: MDQT::CLI::CacheControl.cache_type(options),
         )
 
-        args.empty? ? [client.get_metadata("")] : args.collect {|entity_id| client.get_metadata(entity_id)}
+        args.empty? ? [client.get_metadata("")] : args.collect { |entity_id| client.get_metadata(entity_id) }
 
       end
 
@@ -80,14 +80,14 @@ module MDQT
       end
 
       def output_to_stdout(results, options)
-        results.each {|r| puts output(r)}
+        results.each { |r| puts output(r) }
       end
 
       def output_files(results, options)
         prepare_output_directory(options.save_to)
         results.each do |result|
           main_file = output_file_path(result.filename)
-          open(main_file, 'w') {|f|
+          open(main_file, 'w') { |f|
             f.puts result.data
           }
           yay "Created #{main_file}"

data/lib/mdqt/cli/list.rb

@@ -0,0 +1,65 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class List < Base
+
+      def run
+
+        options.validate = true
+
+        advise_on_xml_signing_support
+
+        halt!("Cannot check a metadata file without XML support: please install additional gems") unless MDQT::Client.verification_available?
+
+        response = get_response
+        result = verify_result(response)
+
+        puts result.entity_ids
+
+      end
+
+      def get_response
+
+        client = MDQT::Client.new(
+          service_url(options),
+          verbose: options.verbose,
+          explain: options.explain ? true : false,
+          tls_risky: options.tls_risky ? true : false,
+          cache_type: MDQT::CLI::CacheControl.cache_type(options),
+        )
+
+        client.get_metadata("")
+
+      end
+
+      def verify_result(result)
+
+        if options.validate
+          halt! "The data for #{result.identifier} is not valid when checked against schema:\n#{result.validation_error}" unless result.valid?
+          btw "Data for #{result.identifier.empty? ? 'aggregate' : result.identifier } has been validated against schema" ## FIXME - needs constistent #label maybe?
+        end
+
+        return result unless options.verify_with
+
+        cert_paths = extract_certificate_paths(options.verify_with)
+
+        halt! "Data from #{options.service} is not signed, cannot verify!" unless result.signed?
+        halt! "The data for #{result.identifier} cannot be verified using #{cert_paths.to_sentence}" unless result.verified_signature?(cert_paths)
+        btw "Data for #{result.identifier.empty? ? 'aggregate' : result.identifier } has been verified using '#{cert_paths.to_sentence}'" ## FIXME - needs constistent #label maybe?
+
+        result
+
+      end
+
+    end
+
+    private
+
+  end
+
+end
+
+

data/lib/mdqt/cli/ln.rb

@@ -0,0 +1,74 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Ln < Base
+
+      def run
+
+        options.validate = true
+
+        advise_on_xml_signing_support
+        halt!("Cannot check a metadata file without XML support: please install additional gems") unless MDQT::Client.verification_available?
+
+        client = MDQT::Client.new(
+          options.service,
+          verbose: options.verbose,
+          explain: options.explain ? true : false,
+        )
+
+        args.each do |filename|
+
+          next if File.symlink?(filename)
+
+          file = client.open_metadata(filename)
+
+          halt!("Cannot access file #{filename}") unless file.readable?
+          halt!("File #{filename} is a metadata aggregate, cannot create entityID hashed link!") if file.aggregate?
+          halt!("XML validation failed for #{filename}:\n#{file.validation_error}") unless file.valid?
+
+          halt!("Cannot find entityID for #{filename}") unless file.entity_id
+
+          linkname = file.linkname
+
+          if filename == linkname
+            if options.force
+              hey("Warning: Cannot link file to itself, skipping! #{filename}")
+              next
+            else
+              halt!("Cannot link file to itself! #{filename}")
+              next
+            end
+            btw("Cannot link file to itself! #{filename}")
+          end
+
+          message = ""
+
+          if File.exists?(linkname)
+            if options.force
+              File.delete(linkname)
+            else
+              old_target = File.readlink(linkname)
+              message = old_target == filename ? "File exists" : "Conflicts with #{filename}"
+              halt!("#{linkname} -> #{old_target} [#{file.entity_id}] #{message}. Use --force to override")
+              next
+            end
+          end
+
+          File.symlink(filename, linkname)
+          hey("#{linkname} -> #{filename} [#{file.entity_id}] #{message}") if options.verbose
+        end
+
+      end
+
+    end
+
+    private
+
+  end
+
+end
+
+

data/lib/mdqt/cli/ls.rb

@@ -0,0 +1,52 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Ls < Base
+
+      def run
+
+        options.validate = true
+
+        advise_on_xml_signing_support
+        halt!("Cannot check a metadata file without XML support: please install additional gems") unless MDQT::Client.verification_available?
+
+        client = MDQT::Client.new(
+          options.service,
+          verbose: options.verbose,
+          explain: options.explain ? true : false,
+        )
+
+        results = []
+
+        args.each do |filename|
+
+          file = client.open_metadata(filename)
+
+          halt!("Cannot access file #{filename}") unless file.readable?
+
+          #halt!("File #{filename} is a metadata aggregate, cannot create entityID hashed link!") if file.aggregate?
+          next if file.aggregate?
+          halt!("XML validation failed for #{filename}:\n#{file.validation_error}") unless file.valid?
+
+          halt!("Cannot find entityID for #{filename}") unless file.entity_id
+
+          results << {id: file.entity_id, type: file.type, filename: file.basename}
+
+        end
+
+        results.sort_by { | r | [r[:id], r[:type]] }.each {|r| puts "#{r[:id]}, #{r[:type]}, #{r[:filename]}" }
+
+      end
+
+    end
+
+    private
+
+  end
+
+end
+
+

data/lib/mdqt/cli/rename.rb

@@ -0,0 +1,60 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Rename < Base
+
+      def run
+
+        options.validate = true
+
+        advise_on_xml_signing_support
+        halt!("Cannot check a metadata file without XML support: please install additional gems") unless MDQT::Client.verification_available?
+
+        client = MDQT::Client.new(
+          options.service,
+          verbose: options.verbose,
+          explain: options.explain ? true : false,
+        )
+
+        args.each do |filename|
+
+          file = client.open_metadata(filename)
+
+          halt!("Cannot access file #{filename}") unless file.readable?
+          halt!("File #{filename} is a metadata aggregate, cannot rename to hashed entityID!") if file.aggregate?
+          halt!("XML validation failed for #{filename}:\n#{file.validation_error}") unless file.valid?
+
+          halt!("Cannot find entityID for #{filename}") unless file.entity_id
+
+          newname = file.linkname # Using the same name as the link, not super-obvious
+          next if filename == newname
+
+          message = ""
+
+          if File.exists?(newname)
+            if options.force
+              File.delete(newname)
+            else
+              halt!("Cannot rename #{filename} to #{newname} - File exists! Use --force to override")
+              next
+            end
+          end
+
+          File.rename(filename, newname)
+          hey("#{filename} renamed to #{newname} [#{file.entity_id}] #{message}") if options.verbose
+        end
+
+      end
+
+    end
+
+    private
+
+  end
+
+end
+
+

data/lib/mdqt/cli/services.rb

@@ -0,0 +1,25 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Services < Base
+
+      def run
+
+        puts "\nKnown services:"
+        puts
+        MDQT::CLI::Defaults.services.each do |service|
+          puts "#{service[:alias]}: #{service[:url]}"
+        end
+        puts
+        puts "Specify these in commands using the --service option or MDQ_BASE_URL environment variable"
+        puts
+      end
+
+    end
+
+  end
+end
+

data/lib/mdqt/cli/url.rb

@@ -0,0 +1,37 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+    require 'uri'
+
+    class URL < Base
+
+      def run
+
+        mds = MDQT::Client::MetadataService.new(service_url(options),
+                                  verbose:false,
+                                  cache_type: :none,
+                                  explain: false,
+                                  tls_cert_check: false)
+
+        if args.empty?
+          puts service_url(options)
+        else
+          args.each do |arg|
+            puts build_url(mds, arg)
+          end
+        end
+
+      end
+
+      def build_url(mds, entity_id)
+
+        URI.join(service_url(options), "/entities/#{mds.prepare_id(entity_id)}")
+      end
+
+    end
+
+  end
+end
+

data/lib/mdqt/cli.rb

@@ -4,13 +4,20 @@ module MDQT
     require 'mdqt/client'
 
     require 'mdqt/cli/defaults'
+    require 'mdqt/cli/cache_control'
 
     require 'mdqt/cli/get'
     require 'mdqt/cli/reset'
     require 'mdqt/cli/version'
     require 'mdqt/cli/transform'
     require 'mdqt/cli/check'
-
+    require 'mdqt/cli/entities'
+    require 'mdqt/cli/ln'
+    require 'mdqt/cli/ls'
+    require 'mdqt/cli/list'
+    require 'mdqt/cli/services'
+    require 'mdqt/cli/rename'
+    require 'mdqt/cli/url'
 
   end
 

data/lib/mdqt/client/metadata_file.rb

@@ -6,8 +6,7 @@ module MDQT
       require 'digest'
 
       def initialize(filename, options = {})
-        @filename = filename
-        @identifier = nil
+        @filename = File.absolute_path(filename)
         @data = nil
         @expires = nil
         @etag = nil
@@ -19,8 +18,21 @@ module MDQT
         @filename
       end
 
+      def basename
+        File.basename(filename)
+      end
+
       def identifier
-        @identifier
+        entity_id
+      end
+
+      def entity_id
+        raise "Incorrect metadata file type - aggregate" if aggregate?
+        @entity_id ||= extract_entity_id
+      end
+
+      def entity_ids
+        @entity_ids ||= extract_entity_ids
       end
 
       def data
@@ -32,7 +44,11 @@ module MDQT
       end
 
       def type
-        @type
+        @type ||= calculate_type
+      end
+
+      def aggregate?
+        type == :aggregate
       end
 
       def expires
@@ -75,13 +91,46 @@ module MDQT
           @filename = "aggregate-#{Digest::SHA1.hexdigest(@service)}.xml"
         else
           @filename ||= identifier.start_with?("{sha1}") ?
-                            "#{@identifier.gsub("{sha1}","")}.xml" :
-                            "#{Digest::SHA1.hexdigest(@identifier)}.xml"
+                          "#{@identifier.gsub("{sha1}", "")}.xml" :
+                          "#{Digest::SHA1.hexdigest(@identifier)}.xml"
+        end
+      end
+
+      def linkname
+        if aggregate?
+          raise "Not supported for aggregates"
+        else
+          "#{Digest::SHA1.hexdigest(entity_id)}.xml"
         end
       end
 
+      def symlink?
+        File.symlink?(filename)
+      end
+
       private
 
+      def calculate_type
+        return :aggregate if data.include?("<EntitiesDescriptor")
+        if data.include?("EntityDescriptor")
+          return :alias if symlink?
+          return :entity
+        end
+        :unknown
+      end
+
+      def xml_doc
+        Nokogiri::XML.parse(data).remove_namespaces!
+      end
+
+      def extract_entity_id
+        xml_doc.xpath("/EntityDescriptor/@entityID").text
+      end
+
+      def extract_entity_ids
+        xml_doc.xpath("//EntityDescriptor/@entityID").map(&:text)
+      end
+
     end
 
   end

data/lib/mdqt/client/metadata_response.rb

@@ -4,15 +4,17 @@ module MDQT
     class MetadataResponse
 
       require 'digest'
+      require 'uri'
 
       def initialize(identifier, service, http_response, options = {})
 
         @requested_identitier = identifier
-        @identifier = URI.decode(identifier)
+        @identifier = URI.decode_www_form_component(identifier)
         @service = service
         @code = http_response.status || 500
         @data = http_response.body || ""
-        @type = http_response.headers['Content-Type']
+        @type = nil
+        @content_type = http_response.headers['Content-Type']
         @expires = http_response.headers['Expires']
         @etag = http_response.headers['ETag']
         @last_modified = http_response.headers['Last-Modified']
@@ -33,10 +35,21 @@ module MDQT
         @identifier
       end
 
+
       def requested_identifier
         @identifier
       end
 
+      def entity_id
+        raise "Incorrect metadata file type - aggregate" if aggregate?
+        @entity_id ||= extract_entity_id
+      end
+
+      def entity_ids
+        @entity_ids ||= extract_entity_ids
+      end
+
+
       def service
         @service
       end
@@ -50,7 +63,11 @@ module MDQT
       end
 
       def type
-        @type
+        @type ||= calculate_type
+      end
+
+      def content_type
+        @content_type
       end
 
       def expires
@@ -135,6 +152,28 @@ module MDQT
 
       private
 
+      def calculate_type
+
+        return :html if data[0,1000].include?('<!doctype html>')
+        return :aggregate if data[0,5000].include?("EntitiesDescriptor")
+        if data[0,5000].include?("EntityDescriptor")
+          return :alias if symlink?
+          return :entity
+        end
+        :unknown
+      end
+
+      def xml_doc
+        Nokogiri::XML.parse(data).remove_namespaces!
+      end
+
+      def extract_entity_id
+        xml_doc.xpath("/EntityDescriptor/@entityID").text
+      end
+
+      def extract_entity_ids
+        xml_doc.xpath("//EntityDescriptor/@entityID").map(&:text)
+      end
 
     end