aaf-mdqt 0.8.0

data/lib/mdqt/cli/ls.rb

@@ -0,0 +1,54 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Ls < Base
+
+      def run
+
+        options.validate = true
+
+        advise_on_xml_signing_support
+        halt!("Cannot check a metadata file without XML support: please install additional gems") unless MDQT::Client.verification_available?
+
+        client = MDQT::Client.new(
+          options.service,
+          verbose: options.verbose,
+          explain: options.explain ? true : false,
+        )
+
+        results = []
+
+        p_args = args.empty? ? Dir.glob("*.xml") : args
+
+        p_args.each do |filename|
+
+          file = client.open_metadata(filename)
+
+          halt!("Cannot access file #{filename}") unless file.readable?
+
+          #halt!("File #{filename} is a metadata aggregate, cannot create entityID hashed link!") if file.aggregate?
+          next if file.aggregate?
+          halt!("XML validation failed for #{filename}:\n#{file.validation_error}") unless file.valid?
+
+          halt!("Cannot find entityID for #{filename}") unless file.entity_id
+
+          results << {id: file.entity_id, type: file.type, filename: file.basename}
+
+        end
+
+        results.sort_by { | r | [r[:id], r[:type]] }.each {|r| puts "#{r[:id]}, #{r[:type]}, #{r[:filename]}" }
+
+      end
+
+    end
+
+    private
+
+  end
+
+end
+
+

data/lib/mdqt/cli/rename.rb

@@ -0,0 +1,75 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Rename < Base
+
+      def run
+
+        options.validate = true
+
+        advise_on_xml_signing_support
+        halt!("Cannot check a metadata file without XML support: please install additional gems") unless MDQT::Client.verification_available?
+
+        client = MDQT::Client.new(
+          options.service,
+          verbose: options.verbose,
+          explain: options.explain ? true : false,
+        )
+
+        halt!("Please specify a file to rename!") if args.empty?
+
+        args.each do |filename|
+
+          next if File.symlink?(filename)
+
+          file = client.open_metadata(filename)
+
+          halt!("Cannot access file #{filename}") unless file.readable?
+          halt!("File #{filename} is a metadata aggregate, cannot rename to hashed entityID!") if file.aggregate?
+          halt!("XML validation failed for #{filename}:\n#{file.validation_error}") unless file.valid?
+
+          halt!("Cannot find entityID for #{filename}") unless file.entity_id
+
+          newname = file.linkname # Using the same name as the link, not super-obvious
+          next if filename == newname
+
+          if file.turd?
+            hey "Warning: will not process backup/turd files"
+            next
+          end
+
+          message = ""
+
+          if File.exist?(newname)
+            if options.force
+              File.delete(newname)
+            else
+              halt!("Cannot rename #{filename} to #{newname} - File exists! Use --force to override")
+              next
+            end
+          end
+
+          File.rename(filename, newname)
+
+          if options.link
+            File.delete(filename) if options.force && File.exist?(filename)
+            File.symlink(newname, filename) unless newname == filename
+          end
+
+          hey("#{filename} renamed to #{newname} [#{file.entity_id}] #{message}") if options.verbose
+        end
+
+      end
+
+    end
+
+    private
+
+  end
+
+end
+
+

data/lib/mdqt/cli/reset.rb

@@ -0,0 +1,27 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Reset < Base
+
+      def run
+
+        client = MDQT::Client.new(
+            :not_required,
+            verbose: false,
+            cache_type: :file
+        )
+
+        print "Removing all cached files... "
+        client.cache_reset!
+        yay "done."
+
+      end
+
+    end
+
+  end
+end
+

data/lib/mdqt/cli/services.rb

@@ -0,0 +1,25 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Services < Base
+
+      def run
+
+        puts "\nKnown services:"
+        puts
+        MDQT::CLI::Defaults.services.each do |service|
+          puts "#{service[:alias]}: #{service[:url]}"
+        end
+        puts
+        puts "Specify these in commands using the --service option or MDQ_BASE_URL environment variable"
+        puts
+      end
+
+    end
+
+  end
+end
+

data/lib/mdqt/cli/transform.rb

@@ -0,0 +1,33 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Transform < Base
+
+      IdentifierUtils = MDQT::Client::IdentifierUtils
+
+      def run
+
+        halt!("No entityIDs have been specified!") if args.empty?
+
+        args.each do |arg|
+          puts transform(arg)
+        end
+
+      end
+
+      def transform(arg)
+        arg = arg.strip
+        return arg if IdentifierUtils.valid_transformed?(arg)
+        return IdentifierUtils.correct_lazy_transformed(arg) if IdentifierUtils.lazy_transformed?(arg)
+        return IdentifierUtils.correct_fish_transformed(arg) if IdentifierUtils.fish_transformed?(arg)
+        IdentifierUtils.transform_uri(arg)
+      end
+
+    end
+
+  end
+end
+

data/lib/mdqt/cli/url.rb

@@ -0,0 +1,37 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+    require 'uri'
+
+    class URL < Base
+
+      def run
+
+        mds = MDQT::Client::MetadataService.new(service_url(options),
+                                  verbose:false,
+                                  cache_type: :none,
+                                  explain: false,
+                                  tls_cert_check: false)
+
+        if args.empty?
+          puts service_url(options)
+        else
+          args.each do |arg|
+            puts build_url(mds, arg)
+          end
+        end
+
+      end
+
+      def build_url(mds, entity_id)
+
+        URI.join(service_url(options), "entities/#{mds.prepare_id(entity_id)}")
+      end
+
+    end
+
+  end
+end
+

data/lib/mdqt/cli/version.rb

@@ -0,0 +1,17 @@
+module MDQT
+
+  class CLI
+
+    require 'mdqt/cli/base'
+
+    class Version < Base
+
+      def run
+        puts "MDQT version #{MDQT::VERSION}"
+      end
+
+    end
+
+  end
+end
+

data/lib/mdqt/cli.rb

@@ -0,0 +1,24 @@
+module MDQT
+  class CLI
+
+    require 'mdqt/client'
+
+    require 'mdqt/cli/defaults'
+    require 'mdqt/cli/cache_control'
+
+    require 'mdqt/cli/get'
+    require 'mdqt/cli/reset'
+    require 'mdqt/cli/version'
+    require 'mdqt/cli/transform'
+    require 'mdqt/cli/check'
+    require 'mdqt/cli/entities'
+    require 'mdqt/cli/ln'
+    require 'mdqt/cli/ls'
+    require 'mdqt/cli/list'
+    require 'mdqt/cli/services'
+    require 'mdqt/cli/rename'
+    require 'mdqt/cli/url'
+
+  end
+
+end

data/lib/mdqt/client/identifier_utils.rb

@@ -0,0 +1,51 @@
+module MDQT
+  class Client
+
+    module IdentifierUtils
+
+      require 'digest'
+
+      class << self
+
+        def uri_to_sha1(uri)
+          raise "An empty string cannot be transformed" if uri.empty?
+          Digest::SHA1.hexdigest(uri.to_s.strip)
+        end
+
+        def transform_uri(uri)
+          "{sha1}#{uri_to_sha1(uri)}"
+        end
+
+        def normalize_to_sha1(identifier)
+          identifier.start_with?("{sha1}") ?
+              identifier.gsub("{sha1}", "") :
+              uri_to_sha1(identifier)
+        end
+
+        def valid_transformed?(string)
+          (string =~ /^[{]sha1[}][0-9a-f]{40}$/i).nil? ? false : true
+        end
+
+        def lazy_transformed?(string)
+          (string =~ /^[\[]sha1[\]][0-9a-f]{40}$/i).nil? ? false : true
+        end
+
+        def fish_transformed?(string)
+          (string =~ /^sha1[0-9a-f]{40}$/i).nil? ? false : true
+        end
+
+        def correct_lazy_transformed(lazy)
+         lazy.gsub("[sha1]", "{sha1}").downcase
+        end
+
+        def correct_fish_transformed(lazy)
+          lazy.gsub("sha1", "{sha1}").downcase
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/mdqt/client/metadata_file.rb

@@ -0,0 +1,144 @@
+module MDQT
+  class Client
+
+    class MetadataFile
+
+      require 'digest'
+
+      def initialize(filename, options = {})
+        @filename = File.absolute_path(filename)
+        @data = nil
+        @expires = nil
+        @etag = nil
+        @last_modified = nil
+        @explanation = {}
+      end
+
+      def filename
+        @filename
+      end
+
+      def basename
+        File.basename(filename)
+      end
+
+      def identifier
+        entity_id
+      end
+
+      def entity_id
+        raise "Incorrect metadata file type - aggregate" if aggregate?
+        @entity_id ||= extract_entity_id
+      end
+
+      def entity_ids
+        @entity_ids ||= extract_entity_ids
+      end
+
+      def data
+        @data ||= File.read(filename)
+      end
+
+      def readable?
+        File.readable?(filename)
+      end
+
+      def turd?
+        return true if basename.end_with?("~")
+        return true if basename.end_with?(".bak")
+        false
+      end
+
+      def type
+        @type ||= calculate_type
+      end
+
+      def aggregate?
+        type == :aggregate
+      end
+
+      def expires
+        @expires
+      end
+
+      def etag
+        @etag
+      end
+
+      def last_modified
+        @last_modified
+      end
+
+      def ok?
+        @ok
+      end
+
+      def signed?
+        @data.include? "Signature" # This is... not great
+      end
+
+      def verified_signature?(certs = [], _ = {})
+        validator = MetadataValidator.new(certs: [certs].flatten)
+        validator.verified_signature?(self)
+      end
+
+      def valid?
+        validator = MetadataValidator.new
+        validator.valid?(self)
+      end
+
+      def validation_error
+        validator = MetadataValidator.new
+        validator.validation_error(self)
+      end
+
+      def canonical_filename
+        if identifier.empty?
+          @filename = "aggregate-#{Digest::SHA1.hexdigest(@service)}.xml"
+        else
+          @filename ||= identifier.start_with?("{sha1}") ?
+                          "#{@identifier.gsub("{sha1}", "")}.xml" :
+                          "#{Digest::SHA1.hexdigest(@identifier)}.xml"
+        end
+      end
+
+      def linkname
+        if aggregate?
+          raise "Not supported for aggregates"
+        else
+          "#{Digest::SHA1.hexdigest(entity_id)}.xml"
+        end
+      end
+
+      def symlink?
+        File.symlink?(filename)
+      end
+
+      private
+
+      def calculate_type
+        return :aggregate if data.include?("<EntitiesDescriptor")
+        if data.include?("EntityDescriptor")
+          return :alias if symlink?
+          return :entity
+        end
+        :unknown
+      end
+
+      def xml_doc
+        Nokogiri::XML.parse(data).remove_namespaces!
+      end
+
+      def extract_entity_id
+        xml_doc.xpath("/EntityDescriptor/@entityID").text
+      end
+
+      def extract_entity_ids
+        xml_doc.xpath("//EntityDescriptor/@entityID").map(&:text)
+      end
+
+    end
+
+  end
+
+end

data/lib/mdqt/client/metadata_response.rb

@@ -0,0 +1,182 @@
+module MDQT
+  class Client
+
+    class MetadataResponse
+
+      require 'digest'
+      require 'uri'
+
+      def initialize(identifier, service, http_response, options = {})
+
+        @requested_identitier = identifier
+        @identifier = URI.decode_www_form_component(identifier)
+        @service = service
+        @code = http_response.status || 500
+        @data = http_response.body || ""
+        @type = nil
+        @content_type = http_response.headers['Content-Type']
+        @expires = http_response.headers['Expires']
+        @etag = http_response.headers['ETag']
+        @last_modified = http_response.headers['Last-Modified']
+        @ok = http_response.success?
+        @explanation = {}
+
+        if options[:explain]
+          @explanation[:url] = http_response.env.url.to_s
+          @explanation[:method] = http_response.env.method.to_s.upcase
+          @explanation[:status] = http_response.status
+          @explanation[:response_headers] = http_response.headers
+          @explanation[:request_headers] = http_response.env.request_headers
+        end
+
+      end
+
+      def identifier
+        @identifier
+      end
+
+
+      def requested_identifier
+        @identifier
+      end
+
+      def entity_id
+        raise "Incorrect metadata file type - aggregate" if aggregate?
+        @entity_id ||= extract_entity_id
+      end
+
+      def entity_ids
+        @entity_ids ||= extract_entity_ids
+      end
+
+
+      def service
+        @service
+      end
+
+      def code
+        @code
+      end
+
+      def data
+        @data || ""
+      end
+
+      def type
+        @type ||= calculate_type
+      end
+
+      def content_type
+        @content_type
+      end
+
+      def expires
+        @expires
+      end
+
+      def etag
+        @etag
+      end
+
+      def last_modified
+        @last_modified
+      end
+
+      def ok?
+        @ok
+      end
+
+      def signed?
+        @data.include? "Signature" # This is... not great
+      end
+
+      def verified_signature?(certs = [], _ = {})
+        return true unless ok?  # CHECK ?
+        validator = MetadataValidator.new(certs: [certs].flatten)
+        validator.verified_signature?(self)
+      end
+
+      def valid?
+        validator = MetadataValidator.new
+        validator.valid?(self)
+      end
+
+      def validation_error
+        validator = MetadataValidator.new
+        validator.validation_error(self)
+      end
+
+      def filename
+        if identifier.empty?
+          @filename = "aggregate-#{Digest::SHA1.hexdigest(@service)}.xml"
+        else
+          @filename ||= identifier.start_with?("{sha1}") ?
+                            "#{@identifier.gsub("{sha1}","")}.xml" :
+                            "#{Digest::SHA1.hexdigest(@identifier)}.xml"
+        end
+      end
+
+      def message
+        case code
+        when 200
+          identifier.empty? ? "[200] OK! Data for aggregate has been downloaded from #{service}" :
+              "[200] OK! Data for '#{identifier}' has been downloaded from #{service}"
+        when 304
+          "[304] OK! Data for '#{identifier}' is already available in a local cache"
+        when 400
+          "[400] The identifier '#{identifier}' ('#{requested_identifier}') is malformed or service URL #{service} is incorrect"
+        when 401
+          "[401] Credentials are incorrect or missing for '#{identifier}'"
+        when 403
+          identifier.empty? ? "[403] The MDQ service at #{service} does not support aggregate downloads" :
+              "[403] You do not have access rights to '#{identifier}' at #{service}"
+        when 404
+          identifier.empty? ? "[404] The MDQ service at #{service} is not responding with aggregated metadata or the correct status" :
+              "[404] Entity metadata for '#{identifier}' was not found at #{service}"
+        when 405
+          "[405] The service at #{service} believes the wrong HTTP method was used. We should have used HTTP GET..."
+        when 406
+          "[406] The requested content type is not available at #{service}"
+        when 500
+          "[500] An error has occurred at #{service}"
+        when 505
+          "[505] The service at #{service} claims our request was using pre-1999 web protocols, not HTTP 1.1 or later"
+        else
+          "[#{code}] Sorry - an unknown error has occurred requesting '#{identifier}' from #{service}.\nPlease report this bug!"
+        end
+      end
+
+      def explanation
+        @explanation
+      end
+
+      private
+
+      def calculate_type
+
+        return :html if data[0,1000].include?('<!doctype html>')
+        return :aggregate if data[0,5000].include?("EntitiesDescriptor")
+        if data[0,5000].include?("EntityDescriptor")
+          return :alias if symlink?
+          return :entity
+        end
+        :unknown
+      end
+
+      def xml_doc
+        Nokogiri::XML.parse(data).remove_namespaces!
+      end
+
+      def extract_entity_id
+        xml_doc.xpath("/EntityDescriptor/@entityID").text
+      end
+
+      def extract_entity_ids
+        xml_doc.xpath("//EntityDescriptor/@entityID").map(&:text)
+      end
+
+    end
+
+  end
+
+end