davinci_crd_test_kit 0.13.0 → 0.14.0

data/lib/davinci_crd_test_kit/server/v2.2.1/discovery/discovery_configuration_test.rb

@@ -0,0 +1,159 @@
+module DaVinciCRDTestKit
+  module V221
+    class DiscoveryConfigurationTest < Inferno::Test
+      title 'Server makes configuration options available'
+      id :crd_v221_discovery_configuration
+      description %(
+        According to the spec:
+
+        > CRD servers SHALL, at minimum, offer configuration options for each
+          type of card they support
+
+        This test verifies that all primary hook services contain at least one
+        valid configuration option for `coverage-info`. Secondary hooks are not
+        expected to return cards, so they are ignored in this test.
+      )
+
+      verifies_requirements 'hl7.fhir.us.davinci-crd_2.2.1@dev-4',
+                            'hl7.fhir.us.davinci-crd_2.2.1@dev-5',
+                            'hl7.fhir.us.davinci-crd_2.2.1@dev-7',
+                            'hl7.fhir.us.davinci-crd_2.2.1@dev-8'
+
+      input :cds_services
+      input :crd_discovery_service_ignore_list,
+            optional: true
+
+      def primary_hooks
+        ['appointment-book', 'order-sign', 'order-dispatch']
+      end
+
+      def primary_hook?(hook)
+        primary_hooks.include? hook
+      end
+
+      def required_fields
+        ['code', 'type', 'name', 'description', 'default']
+      end
+
+      def verify_unique_values(hook, service)
+        unique_fields = ['code', 'name', 'description']
+
+        config_options =
+          service
+            .dig('extension', 'davinci-crd.configuration-options')
+
+        return if config_options.blank?
+
+        unique_fields.each do |unique_field|
+          value_counts =
+            config_options
+              .map { |config_option| config_option[unique_field] }
+              .tally
+
+          duplicate_values =
+            value_counts
+              .select { |_value, count| count > 1 }
+              .keys
+
+          next if duplicate_values.blank?
+
+          duplicate_values_string = duplicate_values.map { |value| "\n- `#{value}`" }.join
+
+          add_message(
+            'error',
+            "Services for hook `#{hook}` contain duplicate values for `#{unique_field}`:" \
+            "#{duplicate_values_string}"
+          )
+        end
+      end
+
+      def verify_required_fields(hook, service)
+        service.dig('extension', 'davinci-crd.configuration-options')&.each do |config_option|
+          required_fields.each do |field_name|
+            unless config_option.key?(field_name)
+              add_message(
+                'error',
+                "Hook `#{hook}` service `#{service['id']}` configuration option `#{config_option['code']}` " \
+                "does not contain `#{field_name}` field"
+              )
+              next
+            end
+
+            next if field_name == 'default'
+
+            next if config_option[field_name].is_a? String
+
+            type = config_option[field_name].class
+            add_message(
+              'error',
+              "Expected hook `#{hook}` service `#{service['id']}` configuration option `#{config_option['code']}` " \
+              "field `#{field_name}` to be a String, but found #{type}"
+            )
+          end
+        end
+      end
+
+      run do
+        ignore_list = crd_discovery_service_ignore_list.to_s.split(',').map(&:strip).reject(&:blank?)
+        services =
+          JSON.parse(cds_services)['services']
+            .reject { |service| ignore_list.include? service['id'] }
+
+        services_by_hook = services.group_by { |service| service['hook'] }
+
+        services_by_hook.each do |hook, hook_services|
+          hook_services.each do |service|
+            verify_required_fields(hook, service)
+            verify_unique_values(hook, service)
+          end
+        end
+
+        primary_hook_services = services_by_hook.slice(*primary_hooks).values.flatten
+
+        assert messages.none? { |message| message[:type] == 'error' },
+               'Some services contain invalid configuration options.'
+
+        omit_if primary_hook_services.blank?, 'No services for primary hooks found'
+
+        services_without_configuration_options =
+          primary_hook_services.reject do |service|
+            service['extension'].present? && service['extension']['davinci-crd.configuration-options'].present?
+          end
+
+        if services_without_configuration_options.present?
+          add_message(
+            'error',
+            'The following services do not contain any configuration options: ' \
+            "#{services_without_configuration_options.map { |service| service['code'] }.join(', ')}"
+          )
+        end
+
+        primary_hook_services.each do |service|
+          coverage_info_config =
+            service.dig('extension', 'davinci-crd.configuration-options')&.find do |config_option|
+              config_option['code'] == 'coverage-info'
+            end
+
+          if coverage_info_config.present?
+            if coverage_info_config['type'] != 'boolean'
+              add_message(
+                'error',
+                "Service `#{service['id']}` `coverage-info` configuration option is not of type boolean"
+              )
+            end
+
+            next
+          end
+
+          add_message(
+            'error',
+            "Service `#{service['id']}` does not contain a `coverage-info` configuration option"
+          )
+        end
+
+        assert messages.none? { |message| message[:type] == 'error' },
+               'Not all primary hook services contain valid configuration options.'
+      end
+    end
+  end
+end

data/lib/davinci_crd_test_kit/server/v2.2.1/discovery/discovery_endpoint_test.rb

@@ -0,0 +1,90 @@
+module DaVinciCRDTestKit
+  module V221
+    class DiscoveryEndpointTest < Inferno::Test
+      title 'Server returns a discovery response'
+      id :crd_v221_discovery_endpoint_test
+      description %(
+        A CDS Service provider must expose its discovery endpoint at `{baseURL}/cds-services`
+        as specified in the [CDS Hooks Specification](https://cds-hooks.hl7.org/2.0/#discovery).
+
+        This test checks that the server responds to a GET request at the following endpoint:
+
+        `GET {baseURL}/cds-services`
+
+        It does this by checking that the server responds with an HTTP OK 200 status code
+        and that the body of the response is a valid JSON object. This test does not
+        inspect the structure and content of the response body to see if it contains the required information.
+        It only checks to see if the RESTful interaction is supported and returns a valid JSON object.
+      )
+
+      input_order :base_url, :authentication_required, :encryption_method, :jwks_kid
+      input :base_url
+      input :authentication_required,
+            title: 'Discovery endpoint requires authentication?',
+            type: 'radio',
+            default: 'no',
+            options: {
+              list_options: [
+                {
+                  label: 'No',
+                  value: 'no'
+                },
+                {
+                  label: 'Yes',
+                  value: 'yes'
+                }
+              ]
+            }
+      input :encryption_method,
+            title: 'JWT Signing Algorithm',
+            description: <<~DESCRIPTION,
+              CDS Hooks recommends ES384 and RS384 for JWT signature verification.
+              Select which method to use.
+            DESCRIPTION
+            type: 'radio',
+            default: 'ES384',
+            options: {
+              list_options: [
+                {
+                  label: 'ES384',
+                  value: 'ES384'
+                },
+                {
+                  label: 'RS384',
+                  value: 'RS384'
+                }
+              ]
+            }
+      input :jwks_kid,
+            title: 'CDS Services JWKS kid',
+            description: <<~DESCRIPTION,
+              The key ID of the JWKS private key to use for signing the JWTs when invoking a CDS service endpoint
+              requiring authentication.
+              Defaults to the first JWK in the list if no kid is supplied.
+            DESCRIPTION
+            optional: true
+      output :cds_services
+
+      run do
+        discovery_url = "#{base_url.chomp('/')}/cds-services"
+        headers = { 'Accept' => 'application/json' }
+
+        if authentication_required == 'yes'
+          token = JwtHelper.build(
+            aud: discovery_url,
+            iss: inferno_base_url,
+            jku: "#{inferno_base_url}/jwks.json",
+            kid: jwks_kid,
+            encryption_method:
+          )
+          headers['Authorization'] = "Bearer #{token}"
+        end
+        get(discovery_url, headers:, tags: [DISCOVERY_TAG])
+        assert_response_status(200)
+        assert_valid_json(request.response_body)
+
+        output cds_services: request.response_body
+      end
+    end
+  end
+end

data/lib/davinci_crd_test_kit/server/v2.2.1/discovery/discovery_prefetch_support_test.rb

@@ -0,0 +1,43 @@
+require_relative '../../server_test_helper'
+
+module DaVinciCRDTestKit
+  module V221
+    class DiscoveryPrefetchSupportTest < Inferno::Test
+      include DaVinciCRDTestKit::ServerTestHelper
+
+      title 'Server advertises prefetch support'
+      id :crd_v221_discovery_prefetch_support
+      description %(
+        This test expects the server to demonstrate support for prefetch queries
+        by providing a prefetch query in at least one of the advertised service
+        in the discovery response.
+      )
+
+      verifies_requirements 'hl7.fhir.us.davinci-crd_2.2.1@found-24'
+
+      input :cds_services
+      input :crd_discovery_service_ignore_list,
+            optional: true
+
+      run do
+        object = parse_json(cds_services)
+        assert object['services'], 'Discovery response did not contain `services`'
+
+        services = object['services']
+        assert services.is_a?(Array), 'Services field of the CDS Discovery response object is not an array.'
+
+        ignored_service_ids = crd_discovery_service_ignore_list.to_s.split(',').map(&:strip).reject(&:blank?)
+        services
+          .select { |service| ignored_service_ids.include?(service['id']) }
+          .each do |service|
+            info "Ignoring service `#{service['id']}` because it is in the ignore list."
+          end
+        services_for_crd_validation = services.reject { |service| ignored_service_ids.include?(service['id']) }
+
+        prefetch_supported = services_for_crd_validation.any? { |service| service['prefetch'].present? }
+
+        skip_if !prefetch_supported, 'No CRD services advertised prefetch support'
+      end
+    end
+  end
+end

data/lib/davinci_crd_test_kit/server/v2.2.1/discovery/discovery_services_validation_test.rb

@@ -0,0 +1,121 @@
+require_relative '../../server_test_helper'
+
+module DaVinciCRDTestKit
+  module V221
+    class DiscoveryServicesValidationTest < Inferno::Test
+      include DaVinciCRDTestKit::ServerTestHelper
+
+      title 'Discovery response contains valid services'
+      id :crd_v221_discovery_services_validation
+      description %(
+        As per the [CDS Hooks Spec](https://cds-hooks.hl7.org/2.0/#response),
+        the response to the discovery endpoint SHALL be an object containing
+        a list of CDS services. If your CDS server hosts no CDS services,
+        the discovery endpoint should return a 200 HTTP response with
+        an empty array of services.
+
+        Each CDS service must contain the following required fields:
+        `hook`, `description`, and `id`.
+
+        Additionally, the [CRD
+        Spec](https://hl7.org/fhir/us/davinci-crd/2.2.1/en/deviations.html#crd-version-declaration)
+        states that "CRD servers SHALL declare at least one supported CRD
+        version for each supported hook" using the `davinci-crd.version`
+        extension.
+
+        This test checks for the presence of the required fields and
+        validates that they are of the correct type.
+
+        CDS services provided by the server to support use cases outside the
+        scope of the CRD specification can be provided in an ignore list input to
+        opt-out of CRD validation.
+
+        The test will be skipped if the server hosts no CRD CDS services.
+      )
+
+      verifies_requirements 'hl7.fhir.us.davinci-crd_2.2.1@dev-1'
+
+      input :cds_services
+      input :crd_discovery_service_ignore_list,
+            title: 'Service ID Ignore List',
+            description: %(
+              In the case that CDS services advertised on this server support
+              use cases outside the scope of the CRD specification, provide a
+              comma-separated list of service IDs to opt-out of CRD validation.
+              Ignored services will not be stored for use in later tests.  If
+              blank, all services are checked and saved.
+            ),
+            optional: true
+      output :appointment_book_service_ids, :encounter_start_service_ids, :encounter_discharge_service_ids,
+             :order_dispatch_service_ids, :order_select_service_ids, :order_sign_service_ids
+
+      EXTENSION_KEY = 'davinci-crd.version'.freeze
+
+      def required_fields
+        {
+          'hook' => String,
+          'description' => String,
+          'id' => String,
+          'extension' => Hash
+        }
+      end
+
+      run do
+        object = parse_json(cds_services)
+        assert object['services'], 'Discovery response did not contain `services`'
+
+        services = object['services']
+        assert services.is_a?(Array), 'Services field of the CDS Discovery response object is not an array.'
+
+        ignored_service_ids = crd_discovery_service_ignore_list.to_s.split(',').map(&:strip).reject(&:blank?)
+        services
+          .select { |service| ignored_service_ids.include?(service['id']) }
+          .each do |service|
+            info "Ignoring service `#{service['id']}` because it is in the ignore list."
+          end
+        services_for_crd_validation = services.reject { |service| ignored_service_ids.include?(service['id']) }
+
+        service_hooks_to_ids = services_for_crd_validation.each_with_object({}) do |service, hash|
+          hash[service['hook']] ||= []
+          hash[service['hook']] << service['id'] if service['id']
+        end
+
+        output appointment_book_service_ids: service_hooks_to_ids['appointment-book']&.join(', '),
+               encounter_start_service_ids: service_hooks_to_ids['encounter-start']&.join(', '),
+               encounter_discharge_service_ids: service_hooks_to_ids['encounter-discharge']&.join(', '),
+               order_dispatch_service_ids: service_hooks_to_ids['order-dispatch']&.join(', '),
+               order_select_service_ids: service_hooks_to_ids['order-select']&.join(', '),
+               order_sign_service_ids: service_hooks_to_ids['order-sign']&.join(', ')
+
+        skip_if services.empty?, 'Server hosts no CDS Services.'
+        skip_if services_for_crd_validation.empty?, 'Ignore list excludes all CDS Services from validation.'
+
+        services_for_crd_validation.each do |service|
+          required_fields.each do |field, type|
+            assert(service[field], "Service `#{service['id']}` did not contain required field: `#{field}`")
+            assert(service[field].is_a?(type), "Service `#{service['id']}`: field `#{field}` is not of type #{type}")
+          end
+
+          assert service['extension'].key?(EXTENSION_KEY),
+                 "Service `#{service['id']}`: does not contain a `#{EXTENSION_KEY}` extension"
+          assert service['extension'][EXTENSION_KEY].is_a?(Array),
+                 "Service `#{service['id']}`: `#{EXTENSION_KEY}` extension is not of type Array"
+          assert service['extension'][EXTENSION_KEY].present?,
+                 "Service `#{service['id']}`: `#{EXTENSION_KEY}` extension is empty"
+          non_string_values = service['extension'][EXTENSION_KEY].reject { |value| value.is_a? String }
+          assert non_string_values.blank?,
+                 "Service `#{service['id']}`: `#{EXTENSION_KEY}` extension contains non-string values: " \
+                 "#{non_string_values.join(', ')}"
+
+          invalid_versions =
+            service['extension'][EXTENSION_KEY]
+              .reject { |version| version.match?(/\A[1-9]\d*\.\d+\Z/) }
+
+          assert invalid_versions.blank?,
+                 "Service `#{service['id']}`: `#{EXTENSION_KEY}` extension contains invalid " \
+                 "version strings: #{invalid_versions.join(', ')}"
+        end
+      end
+    end
+  end
+end

data/lib/davinci_crd_test_kit/server/v2.2.1/interaction/server_invoke_hook_test.rb

@@ -0,0 +1,17 @@
+require_relative '../server_urls'
+require_relative '../../server_abstract_invoke_hook_test'
+
+module DaVinciCRDTestKit
+  module V221
+    class ServerInvokeHookTest < ServerAbstractInvokeHookTest
+      include ServerURLs
+
+      title 'Inferno invokes the selected hook'
+      id :crd_v221_server_invoke_hook_test
+
+      def coverage_info_configuration_supported?
+        true
+      end
+    end
+  end
+end

data/lib/davinci_crd_test_kit/server/v2.2.1/must_support/coverage_information_must_support_test.rb

@@ -0,0 +1,71 @@
+require_relative '../../../cross_suite/cards_identification'
+require_relative '../../../cross_suite/tags'
+
+module DaVinciCRDTestKit
+  module V221
+    class CoverageInformationMustSupportTest < Inferno::Test
+      include DaVinciCRDTestKit::CardsIdentification
+
+      title 'Coverage Information responses demonstrate Must Support elements'
+      id :crd_v221_coverage_information_must_support
+      description <<~DESCRIPTION
+        Checks that the server demonstrated must support coverage for the [Coverage Information](https://hl7.org/fhir/us/davinci-crd/2.2.1/en/cards.html#coverage-information-response-type)
+        extension across the hook responses returned during this test session.
+
+        At least one hook invocation must have returned a Coverage Information system action.
+        Additionally, the set of [coverage-information extension](https://hl7.org/fhir/us/davinci-crd/2.2.1/en/StructureDefinition-ext-coverage-information.html)
+        instances returned across those actions must collectively demonstrate all must support elements.
+
+        If this test fails, re-run the hook tests using request payloads that cause the server to return
+        Coverage Information system actions that cover the full set of must support elements.
+      DESCRIPTION
+
+      ALL_HOOKS = [
+        DaVinciCRDTestKit::APPOINTMENT_BOOK_TAG,
+        DaVinciCRDTestKit::ENCOUNTER_START_TAG,
+        DaVinciCRDTestKit::ENCOUNTER_DISCHARGE_TAG,
+        DaVinciCRDTestKit::ORDER_DISPATCH_TAG,
+        DaVinciCRDTestKit::ORDER_SELECT_TAG,
+        DaVinciCRDTestKit::ORDER_SIGN_TAG
+      ].freeze
+
+      def metadata
+        @metadata ||= YAML.load_file(
+          File.join(__dir__, '..', '..', '..', 'cross_suite', 'coverage-information_stu221_metadata.yml')
+        )
+      end
+
+      class MustSupportMetadataHolder
+        attr_accessor :metadata
+
+        def initialize(metadata)
+          self.metadata = metadata
+        end
+
+        def must_supports
+          @must_supports ||= {
+            extensions: metadata[:must_supports][:extensions] || [],
+            slices: metadata[:must_supports][:slices] || [],
+            elements: metadata[:must_supports][:elements] || []
+          }
+        end
+      end
+
+      run do
+        ALL_HOOKS.each { |hook_tag| load_tagged_requests(hook_tag) }
+        sorted_cards = sorted_cards_from_requests(requests)
+        coverage_info_system_actions = sorted_cards['actions'][COVERAGE_INFORMATION_RESPONSE_TYPE]
+
+        assert coverage_info_system_actions.present?,
+               'Coverage Information system action support not demonstrated.'
+
+        coverage_information_extensions = extract_coverage_information_extensions(sorted_cards)
+        assert_must_support_elements_present(
+          coverage_information_extensions,
+          COVERAGE_INFO_EXT_URL,
+          metadata: MustSupportMetadataHolder.new(metadata)
+        )
+      end
+    end
+  end
+end

data/lib/davinci_crd_test_kit/server/v2.2.1/must_support/coverage_information_system_action_across_hooks_validation_test.rb

@@ -0,0 +1,36 @@
+require_relative '../../server_test_helper'
+
+module DaVinciCRDTestKit
+  module V221
+    class CoverageInformationSystemActionAcrossHooksValidationTest < Inferno::Test
+      include DaVinciCRDTestKit::ServerTestHelper
+
+      title 'Coverage Information system actions are valid across all hooks'
+      id :crd_v221_coverage_info_system_action_across_hooks_validation
+      description %(
+        This test verifies the presence of valid [Coverage Information](https://hl7.org/fhir/us/davinci-crd/STU2/cards.html#coverage-information)
+        system action returned by CRD services across all hooks invoked. It verifies the following for each action:
+        - The action type is `update`.
+        - The resource within the action conforms its respective FHIR profile.
+
+        Additionally, the test examines the `coverage-info` extensions within the resource to ensure that:
+        - Entries referencing differing coverage have distinct `coverage-assertion-ids` and `satisfied-pa-ids`
+        (if present).
+        - Entries referencing the same coverage have the same `coverage-assertion-ids` and `satisfied-pa-ids`
+        (if present).
+
+        The test will be skipped if no valid Coverage Information system actions are returned across all hooks.
+      )
+
+      verifies_requirements 'hl7.fhir.us.davinci-crd_2.2.1@resp-14'
+
+      run do
+        verify_at_least_one_test_passes(
+          self.class.parent.parent.groups,
+          'crd_v221_coverage_info_system_action_validation',
+          'None of the hooks invoked returned valid Coverage Info system actions.'
+        )
+      end
+    end
+  end
+end

data/lib/davinci_crd_test_kit/server/v2.2.1/must_support/supported_us_core_versions_test.rb

@@ -0,0 +1,118 @@
+require 'yaml'
+require_relative '../../resource_extractor'
+require_relative '../../server_test_helper'
+require_relative '../../../cross_suite/tags'
+
+module DaVinciCRDTestKit
+  module V221
+    class SupportedUSCoreVersionsTest < Inferno::Test
+      include DaVinciCRDTestKit::ResourceExtractor
+      include DaVinciCRDTestKit::ServerTestHelper
+
+      title 'Provided resources demonstrate support for all required US Core versions'
+      id :crd_v221_supported_us_core_versions
+      description <<~DESCRIPTION
+        Checks whether the embedded FHIR resources provided across hook requests during this test
+        session demonstrate support for all US Core versions required by the CRD implementation
+        guide: 3.1.1, 6.1.0, and 7.0.0.
+
+        This test does not verify exhaustive support for each US Core version. It inspects FHIR
+        resources included in successful hook request `context` and `prefetch` data and verifies
+        that at least one provided resource validates against a US Core profile for each required
+        version.
+      DESCRIPTION
+
+      verifies_requirements 'hl7.fhir.us.davinci-crd_2.2.1@conf-2'
+
+      US_CORE_VERSIONS = {
+        '3.1.1' => 'v3.1.1',
+        '6.1.0' => 'v6.1.0',
+        '7.0.0' => 'v7.0.0'
+      }.freeze
+
+      GENERATED_METADATA_DIR = File.join(
+        Gem::Specification.find_by_name('us_core_test_kit').gem_dir,
+        'lib', 'us_core_test_kit', 'generated'
+      ).freeze
+
+      run do
+        ALL_HOOK_TAGS.each { |hook_tag| load_tagged_requests(hook_tag) }
+        skip_if requests.empty?, 'No requests were made in a previous test as expected.'
+
+        successful_requests = requests.select { |request| request.status == 200 }
+        skip_if successful_requests.empty?, 'All service requests were unsuccessful.'
+
+        embedded_resources = successful_requests.each.flat_map { |request| resources_from_request(request) }
+
+        skip_if embedded_resources.blank?,
+                'No embedded FHIR resources were found in successful hook requests.'
+
+        version_matches = determine_version_matches(embedded_resources)
+        missing_versions = US_CORE_VERSIONS.keys - version_matches.keys
+
+        missing_versions.each do |version|
+          add_message('error', "Support for US Core #{version} was not demonstrated.")
+        end
+
+        assert missing_versions.empty?,
+               'Support for one or more required US Core versions was not demonstrated.'
+      end
+
+      private
+
+      def determine_version_matches(embedded_resources)
+        embedded_resources.each_with_object({}) do |resource, matches|
+          US_CORE_VERSIONS.each_key do |version|
+            next if matches.key?(version)
+
+            matching_profile = us_core_profiles_by_version
+              .dig(version, resource.resourceType)
+              &.find { |profile_url| resource_valid_for_profile?(resource, profile_url) }
+
+            next unless matching_profile.present?
+
+            matches[version] = matching_profile
+          end
+        end
+      end
+
+      def resource_valid_for_profile?(resource, profile_url)
+        validation_details = []
+        resource_is_valid?(
+          resource:,
+          profile_url:,
+          validator_response_details: validation_details,
+          add_messages_to_runnable: false
+        )
+      end
+
+      def us_core_profiles_by_version
+        @us_core_profiles_by_version ||= US_CORE_VERSIONS.transform_values do |directory|
+          version_profile_metadata(directory).each_with_object(
+            Hash.new { |hash, key| hash[key] = [] }
+          ) do |metadata, profiles_by_resource|
+            profile_identifier = versioned_profile_identifier(metadata)
+            next unless profile_identifier.present?
+
+            profiles_by_resource[metadata[:resource]] << profile_identifier
+          end.transform_values(&:uniq)
+        end
+      end
+
+      def version_profile_metadata(directory)
+        Dir.glob(File.join(GENERATED_METADATA_DIR, directory, '*', 'metadata.yml')).map do |path|
+          YAML.load_file(path, aliases: true)
+        end
+      end
+
+      def versioned_profile_identifier(metadata)
+        resource_type = metadata[:resource]
+        profile_url = metadata[:profile_url]
+        profile_version = metadata[:profile_version]
+        return if resource_type.blank? || profile_url.blank? || profile_version.blank?
+
+        "#{profile_url}|#{profile_version}"
+      end
+    end
+  end
+end