inferno_core 0.6.9 → 0.6.11

data/lib/inferno/apps/cli/templates/lib/%library_name%/example_suite.rb.tt

@@ -0,0 +1,128 @@
+require_relative 'metadata'
+require_relative 'example_suite/patient_group'
+
+module <%= module_name %>
+  class ExampleSuite < Inferno::TestSuite
+
+    id :<%= test_suite_id %>
+    title '<%= title_name %>'
+    short_title '<%= title_name %>'
+
+    # TODO: Update the description below to align with the test suite
+    description <<~DESCRIPTION
+      The Example Test Suite is a testing tool for Health Level 7 (HL7®) Fast Healthcare Interoperability Resources (FHIR®)
+      services seeking to meet the requirements of the API criterion within the Example Certification Program.
+
+      *or*
+
+      The Example Test Suite tests systems for their conformance to the [Example Implementation Guide](https://example.com/example).
+
+      ## Organization
+
+      This test suite is organized into testing scenarios that in sum cover all requirements within the Example criterion.
+      The scenarios are intended to be run in order during certification, but can be run out of order to support testing
+      during development or certification preparation. Some scenarios depend on data collected during previous scenarios
+      to function. In these cases, the scenario description describes these dependencies.
+
+      The first three scenarios require the system under test to demonstrate basic SMART App Launch functionality.
+      The fourth uses a valid token provided during earlier tests to verify support for the Single Patient API as
+      described in the criterion. The fifth verifies support for the Multi Patient API, including Backend Services
+      for authorization. Not all authorization-related requirements are verified in the first three scenarios,
+      and the 'Additional Authorization Tests' verify these additional requirements. The last scenario contains
+      a list of 'attestations' and 'visual inspections' for requirements that could not be verified through automated testing.
+
+      *or*
+
+      This test suite is split into three different categories:
+      - All Patients: FHIR Operation to obtain a detailed set of FHIR resources of diverse resource types pertaining to all patients
+      - Group of Patients: FHIR Operation to obtain a detailed set of FHIR resources of diverse resource types pertaining to all members of a specified Group
+      - System Level Export: FHIR Operation to export data from a FHIR server, whether or not it is associated with a patient
+
+      ## Getting Started
+
+      The best way to learn about how to use these tests is the [Example Test Kit walkthrough](https://example.com/Walkthrough),
+      which demonstrates the tests running against a simulated system.
+
+      To get started with the first group of scenarios, please first register the Inferno client as a SMART App with the following information:
+      - SMART Launch URI: https://example.com/smart/launch
+      - OAuth Redirect URI: https://example.com/smart/redirect
+
+      For the multi-patient API, register Inferno with the following JWK Set Url:
+      - https://example.com/suites/custom/example/.well-known/jwks.json
+
+      *or*
+
+      To get started, if your server supports SMART backend services authorization, please first register Inferno with the following JWK Set URL:
+      - https://example.com/suites/custom/example/.well-known/jwks.json
+
+      Then, run the full Example test suite containing both the SMART Backend Services test group and the Bulk Data Export Tests test group.
+      If your server does not support SMART Backend Services authorization, only run the second test group, Bulk Data Export Tests.
+
+      ## Limitations
+
+      Inferno is unable to determine what requests will result in specific kinds of responses from the server under test
+      (e.g., what will result in Instructions being returned vs. Coverage Information). As a result, the tester must
+      supply the request bodies which will cause the system under test to return the desired response types.
+
+      The ability of an Example Server to request additional FHIR resources is not tested.
+
+      Hook configuration is not tested.
+
+      ## *if applicable:* Certification Requirements
+
+      Systems must pass all tests to qualify for Example certification.
+
+    DESCRIPTION
+
+    # These inputs will be available to all tests in this suite
+    input :url,
+          title: 'FHIR Server Base Url'
+
+    input :credentials,
+          title: 'OAuth Credentials',
+          type: :auth_info,
+          optional: true
+
+    # All FHIR requests in this suite will use this FHIR client
+
+    # All FHIR requests in this suite will use this FHIR client
+    fhir_client do
+      url :url
+      auth_info :credentials
+    end
+
+    # All FHIR validation requests will use this FHIR validator
+    fhir_resource_validator do
+      # igs 'identifier#version' # Use this method for published IGs/versions
+      # igs 'igs/filename.tgz'   # Use this otherwise
+
+      exclude_message do |message|
+        message.message.match?(/\A\S+: \S+: URL value '.*' does not resolve/)
+      end
+    end
+
+    # Tests and TestGroups can be defined inline
+    group do
+      id :capability_statement
+      title 'Capability Statement'
+      description 'See a sample description in the Patient Test Group'
+
+      test do
+        id :capability_statement_read
+        title 'Read CapabilityStatement'
+        description 'Read CapabilityStatement from /metadata endpoint'
+
+        run do
+          fhir_get_capability_statement
+
+          assert_response_status(200)
+          assert_resource_type(:capability_statement)
+        end
+      end
+    end
+
+    # Tests and TestGroups can be written in separate files and then included
+    # using their id
+    group from: :patient_group
+  end
+end

data/lib/inferno/apps/cli/templates/lib/%library_name%/metadata.rb.tt

@@ -5,11 +5,73 @@ module <%= module_name %>
     id :<%= test_kit_id %>
     title '<%= title_name %>'
     description <<~DESCRIPTION
-      This is a big markdown description of the test kit.
+      The Example Test Kit is a testing tool for Health IT systems seeking to meet the requirements of API Criterion within the Example Certification Program.
+
+      *or*
+
+      The Example Test Kit validates the conformance of a server implementation to a specific version of the [Example IG](https://example.com/example).
+      Currently, Inferno can test against implementations of following versions of the Example IG: v1.0.0, v1.3.0, v2.0.0, and v3.0.1.
+
+      <!-- break -->
+
+      ## Getting Started
+
+      Please select which approved version of each standard to use, and click ‘Create Test Session’ to begin testing.
+
+      This test kit includes a [simulated conformant FHIR API](https://inferno.healthit.gov/reference-server/)
+      that can be used to demonstrate success for all tests. This simulated API is open source and is available on GitHub.
+      Visit the [walkthrough](https://example.com/Walkthrough) for a demonstration
+      of using these tests against the provided simulated FHIR API.
+
+      ## Status
+
+      The Example Test Kit is actively developed and updates are released monthly.
+
+      *or*
+
+      These tests are a **DRAFT**. Future versions of these tests may verify other requirements and may change how these requirements are tested.
+
+      ## Conformance
+
+      The test kit currently tests all requirements for the
+      [API Criterion within the Example Certification Program](https://example.com/api-criterion).
+      This includes:
+      - The Lorum IG [v1.0.0](https://example.com/lorum/1.0.0)
+      - The Ipsum IG [v2.0.0](https://example.com/ipsum/2.0.0), [v3.0.1](https://example.com/ipsum/3.0.1)
+      - The Dolor IG [v2.0.2](https://example.com/dolor/2.0.2)
+
+      *or*
+
+      The test kit currently tests the following requirements:
+      - Vel mattis erat semper ut
+      - Suspendisse eget tempor
+      - Nulla eu cursus turpis
+      - Praesent orci diam
+
+
+      ## Repository
+
+      The Example Kit can be
+      [downloaded from its GitHub repository](https://example.com/example-test-kit-repo),
+      where additional resources and documentation are also available to help users get
+      started with the testing process. The repository [Wiki](https://example.com/example-test-kit-repo/wiki/)
+      provides a [FAQ](https://example.com/example-test-kit-repo/wiki/FAQ) for testers,
+      and the [Releases](https://example.com/example-test-kit-repo/releases) page provides information about each new release.
+
+      ## Providing Feedback and Reporting Issues
+
+      We welcome feedback on the tests, including but not limited to the following areas:
+
+      - Validation logic, such as potential bugs, lax checks, and unexpected failures.
+      - Requirements coverage, such as requirements that have been missed, tests that necessitate features that the IG does not require, or other issues with the interpretation of the IG’s requirements.
+      - User experience, such as confusing or missing information in the test UI.
+
+      Please report any issues with this set of tests in the [issues section](https://example.com/example-test-kit-repo/issues) of the repository.
     DESCRIPTION
+
     suite_ids [:<%= test_suite_id %>]
-    # tags ['SMART App Launch', 'US Core']
-    # last_updated '2024-03-07'
+    tags [] # E.g., ['SMART App Launch', 'US Core']
+    last_updated LAST_UPDATED
     version VERSION
     maturity 'Low'
     authors <%= authors %>

data/lib/inferno/apps/cli/templates/lib/%library_name%/version.rb.tt

@@ -1,3 +1,4 @@
 module <%= module_name %>
   VERSION = '0.0.0'.freeze
+  LAST_UPDATED = '<%= Date.today.strftime("%Y-%m-%d") %>'.freeze # TODO: update next release
 end

data/lib/inferno/apps/cli/templates/lib/%library_name%.rb.tt

@@ -1 +1 @@
-require_relative '<%= library_name %>/suite'
+require_relative '<%= library_name %>/example_suite'

data/lib/inferno/apps/web/router.rb

@@ -1,4 +1,5 @@
 require 'erb'
+require_relative '../../feature'
 
 Dir.glob(File.join(__dir__, 'controllers', '**', '*.rb')).each { |path| require_relative path }
 
@@ -47,13 +48,17 @@ module Inferno
           put '/:id/check_configuration',
               to: Inferno::Web::Controllers::TestSuites::CheckConfiguration,
               as: :check_configuration
-          get ':id/requirements',
-              to: Inferno::Web::Controllers::TestSuites::Requirements::Index,
-              as: :requirements
+          if Feature.requirements_enabled?
+            get ':id/requirements',
+                to: Inferno::Web::Controllers::TestSuites::Requirements::Index,
+                as: :requirements
+          end
         end
 
-        scope 'requirements' do
-          get '/:id', to: Inferno::Web::Controllers::Requirements::Show, as: :show
+        if Feature.requirements_enabled?
+          scope 'requirements' do
+            get '/:id', to: Inferno::Web::Controllers::Requirements::Show, as: :show
+          end
         end
 
         get '/requests/:id', to: Inferno::Web::Controllers::Requests::Show, as: :requests_show

data/lib/inferno/apps/web/serializers/input.rb

@@ -14,6 +14,7 @@ module Inferno
         field :optional, if: :field_present?
         field :options, if: :field_present?
         field :locked, if: :field_present?
+        field :hidden, if: :field_present?
         field :value, if: :field_present?
       end
     end

data/lib/inferno/apps/web/serializers/serializer.rb

@@ -1,5 +1,6 @@
 require 'oj'
 require 'blueprinter'
+require_relative '../../../feature'
 
 module Inferno
   module Web
@@ -13,6 +14,12 @@ module Inferno
             result.send(name).present?
           end
         end
+
+        # When removing the feature flag, replace all instances of this method
+        # with `.field_present?`
+        def self.field_present_and_requirements_enabled?(field_name, result, options)
+          field_present?(field_name, result, options) && Feature.requirements_enabled?
+        end
       end
     end
   end

data/lib/inferno/apps/web/serializers/test.rb

@@ -20,6 +20,7 @@ module Inferno
         field :input_instructions
         field :user_runnable?, name: :user_runnable
         field :optional?, name: :optional
+        field :verifies_requirements, if: :field_present_and_requirements_enabled?
       end
     end
   end

data/lib/inferno/apps/web/serializers/test_group.rb

@@ -32,6 +32,7 @@ module Inferno
           Input.render_as_hash(group.available_inputs(suite_options).values)
         end
         field :output_definitions, name: :outputs, extractor: HashValueExtractor
+        field :verifies_requirements, if: :field_present_and_requirements_enabled?
       end
     end
   end

data/lib/inferno/apps/web/serializers/test_suite.rb

@@ -38,7 +38,7 @@ module Inferno
             suite_options = options[:suite_options]
             Input.render_as_hash(suite.available_inputs(suite_options).values)
           end
-          field :requirement_sets, if: :field_present? do |suite, options|
+          field :requirement_sets, if: :field_present_and_requirements_enabled? do |suite, options|
             selected_options = options[:suite_options] || []
             requirement_sets = suite.requirement_sets.select do |requirement_set|
               requirement_set.suite_options.all? { |suite_option| selected_options.include? suite_option }
@@ -46,6 +46,7 @@ module Inferno
 
             RequirementSet.render_as_hash(requirement_sets)
           end
+          field :verifies_requirements, if: :field_present_and_requirements_enabled?
         end
       end
     end

data/lib/inferno/config/boot/suites.rb

@@ -45,6 +45,9 @@ Inferno::Application.register_provider(:suites) do
       if descendant.id.blank? || descendant.id == 'Inferno::Entities::TestSuite'
         raise StandardError, "Error initializing test suite #{descendant.name}: test suite ID is not set"
       end
+
+      # This will lock the short IDs if a short ID map for this suite is present
+      descendant.assign_short_ids
     end
   end
 end

data/lib/inferno/dsl/fhir_evaluation/default.yml

@@ -0,0 +1,68 @@
+Environment:
+  ExternalValidator: 
+    Enabled: false
+    Url: ''
+  VSAC:
+    Username: 'apikey'
+    Password: ''
+
+Rule:
+  AllReferencesResolve: 
+    Description: 'All References in Examples resolve to each other'
+    Enabled: true
+  AllResourcesReachable:
+    Description: 'All resources in Examples are reachable'
+    Enabled: true
+  AllMustSupportsPresent:
+    Description: 'An instance of all MustSupport elements, extensions, and slices is present in the given resources'
+    Enabled: true
+    # RequirementExtensionUrl accepts an extension URL which tags element definitions as required for the purposes of this test, even if not Must Support.
+    # For instance, US Core elements with the "http://hl7.org/fhir/us/core/StructureDefinition/uscdi-requirement" extension
+    # may be considered required even if not necessarily tagged Must Support.
+    # An instance of the extension must have valueBoolean=true to be recognized.
+    RequirementExtensionUrl: null
+
+    # Set WriteMetadataForDebugging:true to have the test write out the metadata it used to a temporary file
+    WriteMetadataForDebugging: false
+
+    ConformanceOptions:
+      # ConformanceOptions allows selecting from a few approaches to determine which subset of resources
+      # should be used to search for the MustSupport elements from each profile.
+      # Resources that are not the same type as the target of the profile are never searched, regardless of option.
+
+      # - If considerMetaProfile, the search will include resources that declare the current profile in meta.profile
+      considerMetaProfile: true
+
+      # - If considerValidationResults, resources will be validated against each profile to determine which they should be checked against.
+      #   The search will include resources that validate against the current profile
+      #   (in other words, resources for which a validation request produces no errors).
+      considerValidationResults: false
+
+      # - If considerOnlyResourceType, the search will include resources of the same type as the profile target type (StructureDefintion.type)
+      considerOnlyResourceType: false
+  AllExtensionsUsed:
+    Description: 'All extensions specified in profiles are represented in Examples'
+    Enabled: true
+  DifferentialContentHasExamples:
+    Enabled: true
+  ValueSetsDemonstrate:
+    Descripton: 'Examples demonstrate reasonable coverage of valuesets defines in an IG. (Note this probably only makes sense for small valuesets such as status options, not something like disease codes from SNOMED)'
+    Enabled: true
+    IgnoreUnloadableValueset: false
+    Exclude:
+      URN: true # Exclude if system is provided as Uniform Resource Name "urn:"
+      Filter: true # Exclude filter
+      SystemOnly: true # Exclude if only system is provided (e.g. http://loing.org)
+  AllSearchParametersHaveExamples:
+    Description: 'Examples cover all search parameters defined in an IG'
+    Enabled: true
+  AllDefinedExtensionsHaveExamples:
+    Description: 'All defined extensions are represented in Examples'
+    Enabled: true
+  AllProfilesHaveExamples:
+    Description: 'All profiles defined in an IG have at least one example instance'
+    Enabled: true
+    ConformanceOptions:
+      considerMetaProfile: true
+      considerValidationResults: false
+      considerOnlyResourceType: false

data/lib/inferno/dsl/fhir_evaluation/evaluator.rb

@@ -24,13 +24,11 @@ module Inferno
         def evaluate(data, config = Config.new)
           context = EvaluationContext.new(@ig, data, config, validator)
 
-          active_rules = []
           config.data['Rule'].each do |rulename, rule_details|
-            active_rules << rulename if rule_details['Enabled']
-          end
+            next unless rule_details['Enabled']
 
-          Rule.descendants.each do |rule|
-            rule.new.check(context) if active_rules.include?(rule.name.demodulize)
+            Rule.descendants.select { |rule| rule.name.demodulize == rulename }
+              .each { |rule| rule.new.check(context) }
           end
 
           context.results

data/lib/inferno/dsl/fhir_evaluation/rules/all_defined_extensions_have_examples.rb

@@ -25,10 +25,10 @@ module Inferno
             end
 
             if unused_extensions.any?
-              message = "Found unused extensions defined in the IG: \n\t #{unused_extensions.join(', ')}"
+              message = "Found defined extensions in the IG without examples: \n\t #{unused_extensions.join(', ')}"
               result = EvaluationResult.new(message, rule: self)
             else
-              message = 'All defined extensions are represented in examples'
+              message = 'All defined extensions in the IG have examples.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             end
 

data/lib/inferno/dsl/fhir_evaluation/rules/all_extensions_used.rb

@@ -12,7 +12,7 @@ module Inferno
               message = get_fail_message(unused_extensions)
               result = EvaluationResult.new(message, rule: self)
             else
-              message = 'All extensions specified in profiles are represented in instances.'
+              message = 'All extensions specified in profiles are used in examples.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             end
 
@@ -63,7 +63,7 @@ module Inferno
           end
 
           def get_fail_message(extensions)
-            message = 'Found extensions specified in profiles, but not used in instances:'
+            message = 'Found extensions specified in profiles, but NOT used in examples:'
             extensions.each do |profile, extension|
               message += "\n Profile: #{profile}, \n\tExtensions: #{extension.join(', ')}" unless extension.empty?
             end

data/lib/inferno/dsl/fhir_evaluation/rules/all_must_supports_present.rb

@@ -48,7 +48,7 @@ module Inferno
             else
               message = 'Found Profiles with not all MustSupports represented:'
               missing_items_by_profile.each do |profile_url, missing_items|
-                message += "\n\t\t#{profile_url}: #{missing_items.join(', ')}"
+                message += "\n\t#{profile_url}: #{missing_items.join(', ')}"
               end
               result = EvaluationResult.new(message, rule: self)
             end

data/lib/inferno/dsl/fhir_evaluation/rules/all_profiles_have_examples.rb

@@ -29,7 +29,7 @@ module Inferno
               message = "Found profiles without examples: \n\t #{unused_profile_urls.join(', ')}"
               result = EvaluationResult.new(message, rule: self)
             else
-              message = 'All profiles have example instances.'
+              message = 'All profiles have examples.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             end
 

data/lib/inferno/dsl/fhir_evaluation/rules/all_references_resolve.rb

@@ -29,7 +29,7 @@ module Inferno
               message = gen_reference_fail_message(unresolved_references)
               result = EvaluationResult.new(message, rule: self)
             else
-              message = 'All references resolve'
+              message = 'All references in examples resolve.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             end
 
@@ -44,7 +44,7 @@ module Inferno
               "\n Resource (id): #{resource_id} #{reference_detail}"
             end.join(',')
 
-            "Found unresolved references: #{result_message}"
+            "Found unresolved references in examples: #{result_message}"
           end
         end
       end

data/lib/inferno/dsl/fhir_evaluation/rules/all_resources_reachable.rb

@@ -27,11 +27,11 @@ module Inferno
             island_resources.to_a.sort!
 
             if island_resources.any?
-              message = "Found resources that have no resolved references and are not referenced: #{
+              message = "Found resources in examples have no resolved references and are not referenced: #{
                 island_resources.join(', ')}"
               result = EvaluationResult.new(message, rule: self)
             else
-              message = 'All resources are reachable'
+              message = 'All resources in examples are reachable.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             end
 

data/lib/inferno/dsl/fhir_evaluation/rules/all_search_parameters_have_examples.rb

@@ -23,13 +23,14 @@ module Inferno
             end
 
             if unused_resource_urls.any?
-              message = "Found SearchParameters with no searchable data: \n\t#{unused_resource_urls.join(' ,')}"
+              unused_resource_list = unused_resource_urls.join("\n\t")
+              message = "Found SearchParameters with no searchable data in examples: \n\t#{unused_resource_list}"
               result = EvaluationResult.new(message, rule: self)
             elsif !search_params.empty?
-              message = 'All SearchParameters have examples'
+              message = 'All SearchParameters have examples.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             else
-              message = 'IG contains no SearchParameter'
+              message = 'IG contains no SearchParameter.'
               result = EvaluationResult.new(message, severity: 'information', rule: self)
             end
 
@@ -42,13 +43,13 @@ module Inferno
             # but without it there's no other way to select a value
             # Return warning if params don't include expression
             unless param.expression
-              message = "Search parameter #{param.url} doesn't have an expression."
+              message = "Search parameter #{param.url} doesn't include an expression."
               result = EvaluationResult.new(message, severity: 'warning', rule: self)
               context.add_result result
               return false
             end
 
-            used = false
+            param_used = false
 
             context.data.each do |resource|
               next unless param.base.include? resource.resourceType
@@ -56,21 +57,31 @@ module Inferno
               begin
                 result = evaluate_fhirpath(resource: resource, path: param.expression)
               rescue StandardError => e
-                message = "SearchParameter #{param.url} failed to evaluate due to an error. " \
-                          "Expression: #{param.expression}. #{e}"
-                result = EvaluationResult.new(message)
+                if e.to_s.include? 'Unable to connect to FHIRPath service'
+                  result = EvaluationResult.new(e.to_s, severity: 'error', rule: self)
+                else
+                  message = "SearchParameter #{param.url} failed to evaluate due to an error. " \
+                            "Expression: #{param.expression}. #{e}"
+                  result = EvaluationResult.new(message, severity: 'warning', rule: self)
+                end
+
                 context.add_result result
 
-                used = true
+                param_used = true
                 break
               end
 
               if result.present?
-                used = true
+                param_used = true
                 break
               end
             end
-            used
+            param_used
+          end
+
+          def add_message(message_type, message)
+            # No implementation but to prevent error from evaluate_fhirpath().
+            # Without this, will throw "undefined method" error since it expects to be called from a Runnable.
           end
         end
       end

data/lib/inferno/dsl/fhir_evaluation/rules/differential_content_has_examples.rb

@@ -15,7 +15,7 @@ module Inferno
               message = gen_differential_fail_message(unused_differential)
               result = EvaluationResult.new(message, rule: self)
             else
-              message = 'All differential fields are represented in instances'
+              message = 'All differential fields are used in examples.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             end
 
@@ -110,7 +110,7 @@ module Inferno
           end
 
           def gen_differential_fail_message(unused_differential)
-            "Found fields highlighted in the differential view, but not used in instances: #{
+            "Found fields highlighted in the differential view, but not used in examples: #{
                             unused_differential.map do |url, field|
                               next if field.empty?
 

data/lib/inferno/dsl/fhir_evaluation/rules/value_sets_demonstrate.rb

@@ -72,7 +72,7 @@ module Inferno
           # rubocop:disable Metrics/CyclomaticComplexity
           def create_result_message
             if value_set_unused.none?
-              message = 'All Value sets are used in Examples:'
+              message = 'All ValueSets are used in examples:'
               value_set_used.map { |value_set| message += "\n\t#{value_set}" }
 
               if value_set_unevaluated.any?
@@ -82,14 +82,14 @@ module Inferno
 
               EvaluationResult.new(message, severity: 'success', rule: self)
             else
-              message = 'Value sets with all codes used at least once in Examples:'
+              message = 'Found ValueSets with all codes used (at least once) in examples:'
               value_set_used.map { |url| message += "\n\t#{url}" }
 
-              message += "\nFound unused Value Sets: "
+              message += "\nFound unused ValueSets: "
               value_set_unused.map { |url| message += "\n\t#{url}" }
 
               if value_set_unevaluated.any?
-                message += "\nFound unevaluated Value Sets: "
+                message += "\nFound unevaluated ValueSets: "
                 value_set_unevaluated.map { |url| message += "\n\t#{url}" }
               end