inferno_core 0.6.8 → 0.6.10

data/lib/inferno/apps/cli/templates/lib/%library_name%/metadata.rb.tt

@@ -5,11 +5,73 @@ module <%= module_name %>
     id :<%= test_kit_id %>
     title '<%= title_name %>'
     description <<~DESCRIPTION
-      This is a big markdown description of the test kit.
+      The Example Test Kit is a testing tool for Health IT systems seeking to meet the requirements of API Criterion within the Example Certification Program.
+
+      *or*
+
+      The Example Test Kit validates the conformance of a server implementation to a specific version of the [Example IG](https://example.com/example).
+      Currently, Inferno can test against implementations of following versions of the Example IG: v1.0.0, v1.3.0, v2.0.0, and v3.0.1.
+
+      <!-- break -->
+
+      ## Getting Started
+
+      Please select which approved version of each standard to use, and click ‘Create Test Session’ to begin testing.
+
+      This test kit includes a [simulated conformant FHIR API](https://inferno.healthit.gov/reference-server/)
+      that can be used to demonstrate success for all tests. This simulated API is open source and is available on GitHub.
+      Visit the [walkthrough](https://example.com/Walkthrough) for a demonstration
+      of using these tests against the provided simulated FHIR API.
+
+      ## Status
+
+      The Example Test Kit is actively developed and updates are released monthly.
+
+      *or*
+
+      These tests are a **DRAFT**. Future versions of these tests may verify other requirements and may change how these requirements are tested.
+
+      ## Conformance
+
+      The test kit currently tests all requirements for the
+      [API Criterion within the Example Certification Program](https://example.com/api-criterion).
+      This includes:
+      - The Lorum IG [v1.0.0](https://example.com/lorum/1.0.0)
+      - The Ipsum IG [v2.0.0](https://example.com/ipsum/2.0.0), [v3.0.1](https://example.com/ipsum/3.0.1)
+      - The Dolor IG [v2.0.2](https://example.com/dolor/2.0.2)
+
+      *or*
+
+      The test kit currently tests the following requirements:
+      - Vel mattis erat semper ut
+      - Suspendisse eget tempor
+      - Nulla eu cursus turpis
+      - Praesent orci diam
+
+
+      ## Repository
+
+      The Example Kit can be
+      [downloaded from its GitHub repository](https://example.com/example-test-kit-repo),
+      where additional resources and documentation are also available to help users get
+      started with the testing process. The repository [Wiki](https://example.com/example-test-kit-repo/wiki/)
+      provides a [FAQ](https://example.com/example-test-kit-repo/wiki/FAQ) for testers,
+      and the [Releases](https://example.com/example-test-kit-repo/releases) page provides information about each new release.
+
+      ## Providing Feedback and Reporting Issues
+
+      We welcome feedback on the tests, including but not limited to the following areas:
+
+      - Validation logic, such as potential bugs, lax checks, and unexpected failures.
+      - Requirements coverage, such as requirements that have been missed, tests that necessitate features that the IG does not require, or other issues with the interpretation of the IG’s requirements.
+      - User experience, such as confusing or missing information in the test UI.
+
+      Please report any issues with this set of tests in the [issues section](https://example.com/example-test-kit-repo/issues) of the repository.
     DESCRIPTION
+
     suite_ids [:<%= test_suite_id %>]
-    # tags ['SMART App Launch', 'US Core']
-    # last_updated '2024-03-07'
+    tags [] # E.g., ['SMART App Launch', 'US Core']
+    last_updated LAST_UPDATED
     version VERSION
     maturity 'Low'
     authors <%= authors %>

data/lib/inferno/apps/cli/templates/lib/%library_name%/suite.rb.tt

@@ -13,13 +13,13 @@ module <%= module_name %>
 
     input :credentials,
           title: 'OAuth Credentials',
-          type: :oauth_credentials,
+          type: :auth_info,
           optional: true
 
     # All FHIR requests in this suite will use this FHIR client
     fhir_client do
       url :url
-      oauth_credentials :credentials
+      auth_info :credentials
     end
 
     # All FHIR validation requests will use this FHIR validator

data/lib/inferno/apps/cli/templates/lib/%library_name%/version.rb.tt

@@ -1,3 +1,4 @@
 module <%= module_name %>
   VERSION = '0.0.0'.freeze
+  LAST_UPDATED = '<%= Date.today.strftime("%Y-%m-%d") %>'.freeze # TODO: update next release
 end

data/lib/inferno/apps/cli/templates/lib/%library_name%.rb.tt

@@ -1 +1 @@
-require_relative '<%= library_name %>/suite'
+require_relative '<%= library_name %>/example_suite'

data/lib/inferno/apps/web/controllers/requirements/show.rb

@@ -0,0 +1,18 @@
+require_relative '../../serializers/requirement'
+
+module Inferno
+  module Web
+    module Controllers
+      module Requirements
+        class Show < Controller
+          def handle(req, res)
+            requirement = repo.find(req.params[:id])
+            halt 404 if requirement.nil?
+
+            res.body = serialize(requirement)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inferno/apps/web/controllers/test_suites/requirements/index.rb

@@ -0,0 +1,29 @@
+module Inferno
+  module Web
+    module Controllers
+      module TestSuites
+        module Requirements
+          class Index < Controller
+            include Import[test_suites_repo: 'inferno.repositories.test_suites']
+            include Import[test_sessions_repo: 'inferno.repositories.test_sessions']
+
+            def handle(req, res)
+              test_suite = test_suites_repo.find(req.params[:id])
+              halt 404, "Test Suite `#{req.params[:id]}` not found" if test_suite.nil?
+
+              test_session = nil
+              if req.params[:session_id]
+                test_session = test_sessions_repo.find(req.params[:session_id])
+                halt 404, "Test session `#{req.params[:session_id]}` not found" if test_session.nil?
+              end
+
+              requirements = repo.requirements_for_suite(test_suite.id, test_session&.id)
+
+              res.body = serialize(requirements)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inferno/apps/web/router.rb

@@ -47,6 +47,13 @@ module Inferno
           put '/:id/check_configuration',
               to: Inferno::Web::Controllers::TestSuites::CheckConfiguration,
               as: :check_configuration
+          get ':id/requirements',
+              to: Inferno::Web::Controllers::TestSuites::Requirements::Index,
+              as: :requirements
+        end
+
+        scope 'requirements' do
+          get '/:id', to: Inferno::Web::Controllers::Requirements::Show, as: :show
         end
 
         get '/requests/:id', to: Inferno::Web::Controllers::Requests::Show, as: :requests_show

data/lib/inferno/apps/web/serializers/input.rb

@@ -14,6 +14,7 @@ module Inferno
         field :optional, if: :field_present?
         field :options, if: :field_present?
         field :locked, if: :field_present?
+        field :hidden, if: :field_present?
         field :value, if: :field_present?
       end
     end

data/lib/inferno/apps/web/serializers/requirement.rb

@@ -0,0 +1,18 @@
+require_relative 'serializer'
+
+module Inferno
+  module Web
+    module Serializers
+      class Requirement < Serializer
+        identifier :id
+
+        field :requirement
+        field :conformance
+        field :actor
+        field :sub_requirements
+        field :conditionality
+        field :url, if: :field_present?
+      end
+    end
+  end
+end

data/lib/inferno/apps/web/serializers/requirement_set.rb

@@ -0,0 +1,13 @@
+require_relative 'serializer'
+
+module Inferno
+  module Web
+    module Serializers
+      class RequirementSet < Serializer
+        identifier :identifier
+
+        field :title
+      end
+    end
+  end
+end

data/lib/inferno/apps/web/serializers/test.rb

@@ -20,6 +20,7 @@ module Inferno
         field :input_instructions
         field :user_runnable?, name: :user_runnable
         field :optional?, name: :optional
+        field :verifies_requirements, if: :field_present?
       end
     end
   end

data/lib/inferno/apps/web/serializers/test_group.rb

@@ -32,6 +32,7 @@ module Inferno
           Input.render_as_hash(group.available_inputs(suite_options).values)
         end
         field :output_definitions, name: :outputs, extractor: HashValueExtractor
+        field :verifies_requirements, if: :field_present?
       end
     end
   end

data/lib/inferno/apps/web/serializers/test_suite.rb

@@ -1,4 +1,5 @@
 require_relative 'preset'
+require_relative 'requirement_set'
 require_relative 'suite_option'
 require_relative 'test_group'
 
@@ -27,6 +28,7 @@ module Inferno
 
         view :full do
           include_view :summary
+
           field :test_groups do |suite, options|
             suite_options = options[:suite_options]
             TestGroup.render_as_hash(suite.groups(suite_options), suite_options:)
@@ -36,6 +38,15 @@ module Inferno
             suite_options = options[:suite_options]
             Input.render_as_hash(suite.available_inputs(suite_options).values)
           end
+          field :requirement_sets, if: :field_present? do |suite, options|
+            selected_options = options[:suite_options] || []
+            requirement_sets = suite.requirement_sets.select do |requirement_set|
+              requirement_set.suite_options.all? { |suite_option| selected_options.include? suite_option }
+            end
+
+            RequirementSet.render_as_hash(requirement_sets)
+          end
+          field :verifies_requirements, if: :field_present?
         end
       end
     end

data/lib/inferno/config/boot/requirements.rb

@@ -0,0 +1,40 @@
+require_relative '../../repositories/requirements'
+
+Inferno::Application.register_provider(:requirements) do
+  prepare do
+    target_container.start :suites
+
+    requirements_repo = Inferno::Repositories::Requirements.new
+
+    test_kit_gems =
+      Bundler
+        .definition
+        .specs
+        .select { |spec| spec.metadata.fetch('inferno_test_kit', 'false').casecmp? 'true' }
+
+    files_to_load = Dir.glob(['lib/*test_kit/requirements/*.csv'])
+
+    if ENV['LOAD_DEV_SUITES'].present?
+      ENV['LOAD_DEV_SUITES'].split(',').map(&:strip).reject(&:empty?).each do |suite|
+        files_to_load.concat Dir.glob(File.join(Inferno::Application.root, 'dev_suites', suite, 'requirements',
+                                                '*.csv'))
+      end
+    end
+
+    files_to_load +=
+      test_kit_gems.flat_map do |gem|
+        [
+          Dir.glob([File.join(gem.full_gem_path, 'lib', '*test_kit', 'requirements', '*.csv')])
+        ].flatten
+      end
+
+    files_to_load.compact!
+    files_to_load.reject! { |file| file.include?('out_of_scope') }
+    files_to_load.map! { |path| File.realpath(path) }
+    files_to_load.uniq!
+
+    files_to_load.each do |path|
+      requirements_repo.insert_from_file(path)
+    end
+  end
+end

data/lib/inferno/config/boot/suites.rb

@@ -45,6 +45,9 @@ Inferno::Application.register_provider(:suites) do
       if descendant.id.blank? || descendant.id == 'Inferno::Entities::TestSuite'
         raise StandardError, "Error initializing test suite #{descendant.name}: test suite ID is not set"
       end
+
+      # This will lock the short IDs if a short ID map for this suite is present
+      descendant.assign_short_ids
     end
   end
 end

data/lib/inferno/dsl/fhir_evaluation/default.yml

@@ -0,0 +1,68 @@
+Environment:
+  ExternalValidator: 
+    Enabled: false
+    Url: ''
+  VSAC:
+    Username: 'apikey'
+    Password: ''
+
+Rule:
+  AllReferencesResolve: 
+    Description: 'All References in Examples resolve to each other'
+    Enabled: true
+  AllResourcesReachable:
+    Description: 'All resources in Examples are reachable'
+    Enabled: true
+  AllMustSupportsPresent:
+    Description: 'An instance of all MustSupport elements, extensions, and slices is present in the given resources'
+    Enabled: true
+    # RequirementExtensionUrl accepts an extension URL which tags element definitions as required for the purposes of this test, even if not Must Support.
+    # For instance, US Core elements with the "http://hl7.org/fhir/us/core/StructureDefinition/uscdi-requirement" extension
+    # may be considered required even if not necessarily tagged Must Support.
+    # An instance of the extension must have valueBoolean=true to be recognized.
+    RequirementExtensionUrl: null
+
+    # Set WriteMetadataForDebugging:true to have the test write out the metadata it used to a temporary file
+    WriteMetadataForDebugging: false
+
+    ConformanceOptions:
+      # ConformanceOptions allows selecting from a few approaches to determine which subset of resources
+      # should be used to search for the MustSupport elements from each profile.
+      # Resources that are not the same type as the target of the profile are never searched, regardless of option.
+
+      # - If considerMetaProfile, the search will include resources that declare the current profile in meta.profile
+      considerMetaProfile: true
+
+      # - If considerValidationResults, resources will be validated against each profile to determine which they should be checked against.
+      #   The search will include resources that validate against the current profile
+      #   (in other words, resources for which a validation request produces no errors).
+      considerValidationResults: false
+
+      # - If considerOnlyResourceType, the search will include resources of the same type as the profile target type (StructureDefintion.type)
+      considerOnlyResourceType: false
+  AllExtensionsUsed:
+    Description: 'All extensions specified in profiles are represented in Examples'
+    Enabled: true
+  DifferentialContentHasExamples:
+    Enabled: true
+  ValueSetsDemonstrate:
+    Descripton: 'Examples demonstrate reasonable coverage of valuesets defines in an IG. (Note this probably only makes sense for small valuesets such as status options, not something like disease codes from SNOMED)'
+    Enabled: true
+    IgnoreUnloadableValueset: false
+    Exclude:
+      URN: true # Exclude if system is provided as Uniform Resource Name "urn:"
+      Filter: true # Exclude filter
+      SystemOnly: true # Exclude if only system is provided (e.g. http://loing.org)
+  AllSearchParametersHaveExamples:
+    Description: 'Examples cover all search parameters defined in an IG'
+    Enabled: true
+  AllDefinedExtensionsHaveExamples:
+    Description: 'All defined extensions are represented in Examples'
+    Enabled: true
+  AllProfilesHaveExamples:
+    Description: 'All profiles defined in an IG have at least one example instance'
+    Enabled: true
+    ConformanceOptions:
+      considerMetaProfile: true
+      considerValidationResults: false
+      considerOnlyResourceType: false

data/lib/inferno/dsl/fhir_evaluation/evaluator.rb

@@ -24,13 +24,11 @@ module Inferno
         def evaluate(data, config = Config.new)
           context = EvaluationContext.new(@ig, data, config, validator)
 
-          active_rules = []
           config.data['Rule'].each do |rulename, rule_details|
-            active_rules << rulename if rule_details['Enabled']
-          end
+            next unless rule_details['Enabled']
 
-          Rule.descendants.each do |rule|
-            rule.new.check(context) if active_rules.include?(rule.name.demodulize)
+            Rule.descendants.select { |rule| rule.name.demodulize == rulename }
+              .each { |rule| rule.new.check(context) }
           end
 
           context.results

data/lib/inferno/dsl/fhir_evaluation/rules/all_defined_extensions_have_examples.rb

@@ -25,10 +25,10 @@ module Inferno
             end
 
             if unused_extensions.any?
-              message = "Found unused extensions defined in the IG: \n\t #{unused_extensions.join(', ')}"
+              message = "Found defined extensions in the IG without examples: \n\t #{unused_extensions.join(', ')}"
               result = EvaluationResult.new(message, rule: self)
             else
-              message = 'All defined extensions are represented in examples'
+              message = 'All defined extensions in the IG have examples.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             end
 

data/lib/inferno/dsl/fhir_evaluation/rules/all_extensions_used.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Inferno
+  module DSL
+    module FHIREvaluation
+      module Rules
+        class AllExtensionsUsed < Rule
+          def check(context)
+            all_extensions = collect_profile_extensions(context.ig.profiles)
+            unused_extensions = remove_found_resource_extensions(all_extensions, context.data)
+            if unused_extensions.any? { |_profile, extensions| !extensions.empty? }
+              message = get_fail_message(unused_extensions)
+              result = EvaluationResult.new(message, rule: self)
+            else
+              message = 'All extensions specified in profiles are used in examples.'
+              result = EvaluationResult.new(message, severity: 'success', rule: self)
+            end
+
+            context.add_result result
+          end
+
+          def collect_profile_extensions(profiles)
+            extensions = Hash.new { |extension, profile| extension[profile] = Set.new }
+            profiles.each do |profile|
+              profile.each_element do |value, metadata|
+                next unless metadata['type'] == 'ElementDefinition'
+
+                path_end = value.id.split('.')[-1]
+                next unless path_end.include?('extension')
+
+                value.type.each do |element_definition|
+                  element_definition.profile.each do |extension_url|
+                    extensions[profile.url].add(extension_url)
+                  end
+                end
+              end
+            end
+            extensions
+          end
+
+          # rubocop:disable Metrics/CyclomaticComplexity
+          def remove_found_resource_extensions(extensions, examples)
+            unused_extensions = extensions.dup
+            examples.each do |resource|
+              resource.each_element do |value, _metadata, path|
+                path_elements = path.split('.')
+                next unless path_elements.length > 1
+
+                next unless path_elements[-2].include?('extension') && path_elements[-1] == 'url'
+
+                profiles = resource&.meta&.profile || []
+                update_unused_extensions(profiles, value, unused_extensions, extensions)
+              end
+            end
+            unused_extensions
+          end
+          # rubocop:enable Metrics/CyclomaticComplexity
+
+          def update_unused_extensions(profiles, value, unused_extensions, extensions)
+            profiles.each do |profile|
+              unused_extensions[profile].delete(value) if extensions.key?(profile)
+            end
+          end
+
+          def get_fail_message(extensions)
+            message = 'Found extensions specified in profiles, but NOT used in examples:'
+            extensions.each do |profile, extension|
+              message += "\n Profile: #{profile}, \n\tExtensions: #{extension.join(', ')}" unless extension.empty?
+            end
+            message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inferno/dsl/fhir_evaluation/rules/all_must_supports_present.rb

@@ -48,7 +48,7 @@ module Inferno
             else
               message = 'Found Profiles with not all MustSupports represented:'
               missing_items_by_profile.each do |profile_url, missing_items|
-                message += "\n\t\t#{profile_url}: #{missing_items.join(', ')}"
+                message += "\n\t#{profile_url}: #{missing_items.join(', ')}"
               end
               result = EvaluationResult.new(message, rule: self)
             end

data/lib/inferno/dsl/fhir_evaluation/rules/all_profiles_have_examples.rb

@@ -29,7 +29,7 @@ module Inferno
               message = "Found profiles without examples: \n\t #{unused_profile_urls.join(', ')}"
               result = EvaluationResult.new(message, rule: self)
             else
-              message = 'All profiles have example instances.'
+              message = 'All profiles have examples.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             end
 

data/lib/inferno/dsl/fhir_evaluation/rules/all_references_resolve.rb

@@ -29,7 +29,7 @@ module Inferno
               message = gen_reference_fail_message(unresolved_references)
               result = EvaluationResult.new(message, rule: self)
             else
-              message = 'All references resolve'
+              message = 'All references in examples resolve.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             end
 
@@ -44,7 +44,7 @@ module Inferno
               "\n Resource (id): #{resource_id} #{reference_detail}"
             end.join(',')
 
-            "Found unresolved references: #{result_message}"
+            "Found unresolved references in examples: #{result_message}"
           end
         end
       end

data/lib/inferno/dsl/fhir_evaluation/rules/all_resources_reachable.rb

@@ -27,11 +27,11 @@ module Inferno
             island_resources.to_a.sort!
 
             if island_resources.any?
-              message = "Found resources that have no resolved references and are not referenced: #{
+              message = "Found resources in examples have no resolved references and are not referenced: #{
                 island_resources.join(', ')}"
               result = EvaluationResult.new(message, rule: self)
             else
-              message = 'All resources are reachable'
+              message = 'All resources in examples are reachable.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             end
 

data/lib/inferno/dsl/fhir_evaluation/rules/all_search_parameters_have_examples.rb

@@ -23,13 +23,14 @@ module Inferno
             end
 
             if unused_resource_urls.any?
-              message = "Found SearchParameters with no searchable data: \n\t#{unused_resource_urls.join(' ,')}"
+              unused_resource_list = unused_resource_urls.join("\n\t")
+              message = "Found SearchParameters with no searchable data in examples: \n\t#{unused_resource_list}"
               result = EvaluationResult.new(message, rule: self)
             elsif !search_params.empty?
-              message = 'All SearchParameters have examples'
+              message = 'All SearchParameters have examples.'
               result = EvaluationResult.new(message, severity: 'success', rule: self)
             else
-              message = 'IG contains no SearchParameter'
+              message = 'IG contains no SearchParameter.'
               result = EvaluationResult.new(message, severity: 'information', rule: self)
             end
 
@@ -42,13 +43,13 @@ module Inferno
             # but without it there's no other way to select a value
             # Return warning if params don't include expression
             unless param.expression
-              message = "Search parameter #{param.url} doesn't have an expression."
+              message = "Search parameter #{param.url} doesn't include an expression."
               result = EvaluationResult.new(message, severity: 'warning', rule: self)
               context.add_result result
               return false
             end
 
-            used = false
+            param_used = false
 
             context.data.each do |resource|
               next unless param.base.include? resource.resourceType
@@ -56,21 +57,31 @@ module Inferno
               begin
                 result = evaluate_fhirpath(resource: resource, path: param.expression)
               rescue StandardError => e
-                message = "SearchParameter #{param.url} failed to evaluate due to an error. " \
-                          "Expression: #{param.expression}. #{e}"
-                result = EvaluationResult.new(message)
+                if e.to_s.include? 'Unable to connect to FHIRPath service'
+                  result = EvaluationResult.new(e.to_s, severity: 'error', rule: self)
+                else
+                  message = "SearchParameter #{param.url} failed to evaluate due to an error. " \
+                            "Expression: #{param.expression}. #{e}"
+                  result = EvaluationResult.new(message, severity: 'warning', rule: self)
+                end
+
                 context.add_result result
 
-                used = true
+                param_used = true
                 break
               end
 
               if result.present?
-                used = true
+                param_used = true
                 break
               end
             end
-            used
+            param_used
+          end
+
+          def add_message(message_type, message)
+            # No implementation but to prevent error from evaluate_fhirpath().
+            # Without this, will throw "undefined method" error since it expects to be called from a Runnable.
           end
         end
       end