inferno_core 0.6.9 → 0.6.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5740a0c7e8d54767851b4f7e9ee7c5c6ca9ac175211d63b4990f99bbbe9266ca
-  data.tar.gz: 4fd1b9c5cb4f37c441a668ae58c8b6cd49aeea18ad393b2d7bbb9699d86a0060
+  metadata.gz: b899985c0f229c1f91aa9218d09f9f502ed315c537398edc911e989cbe61c6a8
+  data.tar.gz: '08cca457d4bb79bdb70487a14fa07f8274dac19faae5623bedef8797541e993b'
 SHA512:
-  metadata.gz: f6adc91f5db1ba2a6535c847c0fe7b691cd85455fb0f34a44bd525428dee3ddcd4c414d7989ea4e13fdd50b9c39778a19d26e97473f1617b85691b4683776f49
-  data.tar.gz: 463d68cf3e187f992bd134bbc177072bfaa9ec625a2006ca1bb23c49591a4bc9521ca027232c68c5d1ebcf499dcca84f3f722ca2afd9bd1b6038bbb19daf4835
+  metadata.gz: 36b765180f9d7168c91238e1c7a63e8cc281f3a795f068e3d6ddb317d419d90dc2401585b8fde37e3d27c0c82fda9ef66dba3c6bd96201f1dafa9d3be301b53f
+  data.tar.gz: d83ffd4c0fec0cf1d3621a7f563c9d369d671bfce5f11c9bdcac9f287c5918a3c37f3ff340cd3429b4dbb1e96e9daa936ce50fe4bd889420cb66c0a41020a777

data/lib/inferno/apps/cli/evaluate/database.yml

@@ -0,0 +1,15 @@
+# Inferno is using `Psych::safe_load` so YAML anchors are disabled
+development:
+  adapter: sqlite
+  database: ':memory:'
+  max_connections: 10
+
+production:
+  adapter: sqlite
+  database: ':memory:'
+  max_connections: 10
+
+test:
+  adapter: sqlite
+  database: ':memory:'
+  max_connections: 10

data/lib/inferno/apps/cli/evaluate/docker-compose.evaluate.yml

@@ -0,0 +1,16 @@
+name: inferno_evaluator_services
+
+services:
+  hl7_validator_service:
+    image: infernocommunity/inferno-resource-validator
+    volumes:
+      - ${TMPDIR}/data/igs:/app/igs
+      # To let the service share your local FHIR package cache,
+      # uncomment the below line
+      # - ~/.fhir:/home/ktor/.fhir
+    ports:
+      - "3501:3500"
+  fhirpath:
+    image: infernocommunity/fhirpath-service
+    ports:
+      - "6790:6789"

data/lib/inferno/apps/cli/evaluate.rb

@@ -2,13 +2,62 @@ require_relative '../../dsl/fhir_evaluation/evaluator'
 require_relative '../../dsl/fhir_evaluation/config'
 require_relative '../../entities'
 require_relative '../../utils/ig_downloader'
+require_relative 'migration'
 
+require 'fileutils'
 require 'tempfile'
 
 module Inferno
   module CLI
-    class Evaluate < Thor::Group
-      def evaluate(ig_path, data_path, _log_level)
+    class Evaluate
+      # @see Inferno::CLI::Main#evaluate
+      def run(ig_path, data_path, options)
+        tmpdir = Dir.mktmpdir
+        Dir.mkdir("#{tmpdir}/data")
+        Dir.mkdir("#{tmpdir}/data/igs")
+        Dir.mkdir("#{tmpdir}/config")
+        FileUtils.cp(File.expand_path('evaluate/database.yml', __dir__), "#{tmpdir}/config/database.yml")
+
+        ENV['TMPDIR'] = tmpdir
+        ENV['FHIRPATH_URL'] = 'http://localhost:6790'
+        ENV['FHIR_RESOURCE_VALIDATOR_URL'] = 'http://localhost:3501'
+
+        puts 'Starting Inferno Evaluator Services...'
+        system("#{services_base_command} up -d #{services_names}")
+
+        ig_path = absolute_path_with_home_expansion(ig_path)
+        data_path = absolute_path_with_home_expansion(data_path) if data_path
+
+        Dir.chdir(tmpdir) do
+          Migration.new.run(Logger::FATAL) # Hide migration output for evaluator
+          evaluate(ig_path, data_path, options)
+        end
+      ensure
+        system("#{services_base_command} down #{services_names}")
+        puts 'Stopped Inferno Evaluator Services'
+
+        FileUtils.remove_entry_secure tmpdir
+      end
+
+      def services_base_command
+        "docker compose -f #{File.join(__dir__, 'evaluate', 'docker-compose.evaluate.yml')}"
+      end
+
+      def services_names
+        'hl7_validator_service fhirpath'
+      end
+
+      # @private
+      def absolute_path_with_home_expansion(path)
+        if path.starts_with? '~'
+          path.sub('~', Dir.home)
+        else
+          File.absolute_path(path)
+        end
+      end
+
+      # @see Inferno::CLI::Main#evaluate
+      def evaluate(ig_path, data_path, options)
         # NOTE: repositories is required here rather than at the top of the file because
         # the tree of requires means that this file and its requires get required by every CLI app.
         # Sequel::Model, used in some repositories, fetches the table schema at instantiation.
@@ -22,7 +71,7 @@ module Inferno
 
         data =
           if data_path
-            DatasetLoader.from_path(File.join(__dir__, data_path))
+            Inferno::DSL::FHIREvaluation::DatasetLoader.from_path(File.join(__dir__, data_path))
           else
             ig.examples
           end
@@ -55,7 +104,6 @@ module Inferno
       def setup_validator(ig_path)
         igs_directory = File.join(Dir.pwd, 'data', 'igs')
         if File.exist?(ig_path) && !File.realpath(ig_path).start_with?(igs_directory)
-          puts "Copying #{File.basename(ig_path)} to data/igs so it is accessible to validator"
           destination_file_path = File.join(igs_directory, File.basename(ig_path))
           FileUtils.copy_file(ig_path, destination_file_path, true)
           ig_path = "igs/#{File.basename(ig_path)}"

data/lib/inferno/apps/cli/execute.rb

@@ -6,6 +6,7 @@ Dir[File.join(__dir__, 'execute', '*_outputter.rb')].each { |outputter| require
 
 module Inferno
   module CLI
+    # @private
     class Execute
       include ::Inferno::Utils::VerifyRunnable
       include ::Inferno::Utils::PersistInputs
@@ -45,7 +46,6 @@ module Inferno
         self.options = options
 
         outputter.print_start_message(self.options)
-
         load_preset_file_and_set_preset_id
 
         results = []
@@ -104,18 +104,6 @@ module Inferno
         @outputter ||= OUTPUTTERS[options[:outputter]].new
       end
 
-      def load_preset_file_and_set_preset_id
-        return unless options[:preset_file]
-        raise StandardError, 'Cannot use `--preset-id` and `--preset-file` options together' if options[:preset_id]
-
-        raise StandardError, "File #{options[:preset_file]} not found" unless File.exist? options[:preset_file]
-
-        options[:preset_id] = JSON.parse(File.read(options[:preset_file]))['id']
-        raise StandardError, "Preset #{options[:preset_file]} is missing id" if options[:preset_id].nil?
-
-        presets_repo.insert_from_file(options[:preset_file])
-      end
-
       def all_selected_groups_and_tests
         @all_selected_groups_and_tests ||= runnables_by_short_id + groups + tests
       end
@@ -123,7 +111,7 @@ module Inferno
       def run_one(runnable, test_run)
         verify_runnable(
           runnable,
-          thor_hash_to_inputs_array(inputs_and_preset),
+          all_inputs,
           test_session.suite_options
         )
 
@@ -132,15 +120,14 @@ module Inferno
         dispatch_job(test_run)
       end
 
-      def inputs_and_preset
+      def all_inputs
         if preset
-          preset_inputs = preset.inputs.to_h do |preset_input|
-            [preset_input[:name], preset_input[:value]]
-          end
+          test_sessions_repo.apply_preset(test_session, preset.id)
+          preset_inputs = session_data_repo.get_all_from_session(test_session.id)
 
-          options.fetch(:inputs, {}).reverse_merge(preset_inputs)
+          thor_hash_to_inputs_array(options.fetch(:inputs, {})) + preset_inputs.map(&:to_hash)
         else
-          options.fetch(:inputs, {})
+          thor_hash_to_inputs_array(options.fetch(:inputs, {}))
         end
       end
 
@@ -159,6 +146,15 @@ module Inferno
         @preset
       end
 
+      def load_preset_file_and_set_preset_id
+        return unless options[:preset_file]
+        raise StandardError, 'Cannot use `--preset-id` and `--preset-file` options together' if options[:preset_id]
+
+        raise StandardError, "File #{options[:preset_file]} not found" unless File.exist? options[:preset_file]
+
+        options[:preset_id] = presets_repo.insert_from_file(options[:preset_file]).id
+      end
+
       def suite
         @suite ||= Inferno::Repositories::TestSuites.new.find(options[:suite])
 
@@ -214,7 +210,7 @@ module Inferno
         {
           test_session_id: test_session.id,
           runnable_id_key(runnable) => runnable.id,
-          inputs: thor_hash_to_inputs_array(inputs_and_preset)
+          inputs: all_inputs
         }
       end
 

data/lib/inferno/apps/cli/main.rb

@@ -1,6 +1,7 @@
 require_relative 'console'
 require_relative 'evaluate'
 require_relative 'migration'
+require_relative 'requirements'
 require_relative 'services'
 require_relative 'suite'
 require_relative 'suites'
@@ -45,7 +46,7 @@ module Inferno
              type: :string,
              desc: 'Export evaluation result to outcome.json as an OperationOutcome'
       def evaluate(ig_path)
-        Evaluate.new.evaluate(ig_path, options[:data_path], Logger::INFO)
+        Evaluate.new.run(ig_path, options[:data_path], options)
       end
 
       desc 'console', 'Start an interactive console session with Inferno'
@@ -59,6 +60,9 @@ module Inferno
         Migration.new.run
       end
 
+      desc 'requirements SUBCOMMAND ...ARGS', 'Perform requirements operations'
+      subcommand 'requirements', Requirements
+
       desc 'start', 'Start Inferno'
       option :watch,
              default: false,

data/lib/inferno/apps/cli/requirements.rb

@@ -0,0 +1,28 @@
+require_relative 'requirements_exporter'
+
+module Inferno
+  module CLI
+    class Requirements < Thor
+      desc 'export_csv', 'Export a CSV represantation of requirements from an excel file'
+      long_desc <<~LONGDESC
+        Creates CSV files for tested requirements and requirements which are not
+        planned to be tested based on the excel files located in
+        "lib/test_kit_name/requirements"
+      LONGDESC
+      def export_csv
+        ENV['NO_DB'] = 'true'
+        RequirementsExporter.new.run
+      end
+
+      desc 'check', 'Check whether the current requirements CSV files are up to date'
+      long_desc <<~LONGDESC
+        Check whether the requirements CSV files are up to date with the excel
+        files in "lib/test_kit_name/requirements"
+      LONGDESC
+      def check
+        ENV['NO_DB'] = 'true'
+        RequirementsExporter.new.run_check
+      end
+    end
+  end
+end

data/lib/inferno/apps/cli/requirements_exporter.rb

@@ -0,0 +1,194 @@
+require 'csv'
+require 'roo'
+
+module Inferno
+  module CLI
+    class RequirementsExporter
+      INPUT_HEADERS =
+        [
+          'ID*',
+          'URL*',
+          'Requirement*',
+          'Conformance*',
+          'Actor*',
+          'Sub-Requirement(s)',
+          'Conditionality',
+          'Verifiable?',
+          'Verifiability Details',
+          'Planning To Test?',
+          'Planning To Test Details'
+        ].freeze
+      REQUIREMENTS_OUTPUT_HEADERS =
+        [
+          'Req Set',
+          'ID',
+          'URL',
+          'Requirement',
+          'Conformance',
+          'Actor',
+          'Sub-Requirement(s)',
+          'Conditionality',
+          'Not Tested Reason',
+          'Not Tested Details'
+        ].freeze
+
+      def local_test_kit_gem
+        @local_test_kit_gem ||= Bundler.definition.specs.find { |spec| spec.full_gem_path == Dir.pwd }
+      end
+
+      def test_kit_name
+        local_test_kit_gem.name
+      end
+
+      def base_requirements_folder
+        @base_requirements_folder ||= Dir.glob(File.join(Dir.pwd, 'lib', '*', 'requirements')).first
+      end
+
+      def requirements_output_file_name
+        "#{test_kit_name}_requirements.csv"
+      end
+
+      def requirements_output_file_path
+        File.join(base_requirements_folder, requirements_output_file_name).freeze
+      end
+
+      def available_input_worksheets
+        @available_input_worksheets ||=
+          Dir.glob(File.join(base_requirements_folder, '*.xlsx'))
+            .reject { |f| f.include?('~$') }
+      end
+
+      def requirement_set_id(worksheet)
+        sheet = worksheet.sheet('Metadata')
+        id_row = sheet.column(1).find_index('Id') + 1
+        sheet.row(id_row)[1]
+      end
+
+      # Of the form:
+      # {
+      #   requirement_set_id_1: [row1, row2, row 3, ...],
+      #   requirement_set_id_2: [row1, row2, row 3, ...]
+      # }
+      def input_requirement_sets
+        requirement_set_hash = Hash.new { |hash, key| hash[key] = [] }
+        available_input_worksheets.each_with_object(requirement_set_hash) do |worksheet_file, requirement_sets|
+          worksheet = Roo::Spreadsheet.open(worksheet_file)
+          set_identifier = requirement_set_id(worksheet)
+
+          CSV.parse(
+            worksheet.sheet('Requirements').to_csv,
+            headers: true
+          ).each do |row|
+            row_hash = row.to_h.slice(*INPUT_HEADERS)
+            row_hash['Sub-Requirement(s)']&.delete_prefix!('mailto:')
+
+            requirement_sets[set_identifier] << row_hash
+          end
+        end
+      end
+
+      def new_requirements_csv # rubocop:disable Metrics/CyclomaticComplexity
+        @new_requirements_csv ||=
+          CSV.generate(+"\xEF\xBB\xBF") do |csv| # start with an unnecessary BOM to make viewing in excel easier
+            csv << REQUIREMENTS_OUTPUT_HEADERS
+
+            input_requirement_sets.each do |requirement_set_id, input_rows|
+              input_rows.each do |row| # NOTE: use row order from source file
+                csv << REQUIREMENTS_OUTPUT_HEADERS.map do |header|
+                  (
+                    case header
+                    when 'Req Set'
+                      requirement_set_id
+                    when 'Not Tested Reason'
+                      if spreadsheet_value_falsy?(row['Verifiable?'])
+                        'Not Verifiable'
+                      elsif spreadsheet_value_falsy?(row['Planning To Test?'])
+                        'Not Tested'
+                      end
+                    when 'Not Tested Details'
+                      if spreadsheet_value_falsy?(row['Verifiable?'])
+                        row['Verifiability Details']
+                      elsif spreadsheet_value_falsy?(row['Planning To Test?'])
+                        row['Planning To Test Details']
+                      end
+                    else
+                      row[header] || row["#{header}*"]
+                    end
+                  )&.strip
+                end
+              end
+            end
+          end
+      end
+
+      def old_requirements_csv
+        @old_requirements_csv ||= File.read(requirements_output_file_path)
+      end
+
+      def run
+        check_presence_of_input_files
+
+        update_requirements =
+          if File.exist?(requirements_output_file_path)
+            if old_requirements_csv == new_requirements_csv
+              puts "'#{requirements_output_file_name}' file is up to date."
+              false
+            else
+              puts 'Requirements set has changed.'
+              true
+            end
+          else
+            puts "No existing #{requirements_output_file_name}."
+            true
+          end
+
+        if update_requirements
+          puts "Writing to file #{requirements_output_file_name}..."
+          File.write(requirements_output_file_path, new_requirements_csv, encoding: Encoding::UTF_8)
+        end
+
+        puts 'Done.'
+      end
+
+      def run_check
+        check_presence_of_input_files
+
+        requirements_ok =
+          if File.exist?(requirements_output_file_path)
+            if old_requirements_csv == new_requirements_csv
+              puts "'#{requirements_output_file_name}' file is up to date."
+              true
+            else
+              puts "#{requirements_output_file_name} file is out of date."
+              false
+            end
+          else
+            puts "No existing #{requirements_output_file_name} file."
+            false
+          end
+
+        return if requirements_ok
+
+        puts <<~MESSAGE
+          Check Failed. To resolve, run:
+
+                bundle exec inferno requirements export_csv
+
+        MESSAGE
+        exit(1)
+      end
+
+      def check_presence_of_input_files
+        return if available_input_worksheets.present?
+
+        puts 'Could not find any input files in directory ' \
+             "#{base_requirements_folder}. Aborting requirements collection."
+        exit(1)
+      end
+
+      def spreadsheet_value_falsy?(string)
+        ['no', 'false'].include? string&.downcase
+      end
+    end
+  end
+end

data/lib/inferno/apps/cli/suite.rb

@@ -52,6 +52,27 @@ module Inferno
 
         puts TTY::Markdown.parse(description)
       end
+
+      desc 'lock_short_ids SUITE_ID', 'Persist the current short_id map for a suite'
+      long_desc <<~LONGDESC
+        Loads the given suite and writes its current short_id map to its corresponding YAML file.
+      LONGDESC
+      def lock_short_ids(suite_id)
+        ENV['NO_DB'] = 'true'
+        Inferno::Application.start(:suites)
+
+        suite = Inferno::Repositories::TestSuites.new.find(suite_id)
+
+        if suite.blank?
+          message = "No suite found with id `#{suite_id}`. Run `inferno suites` to see a list of available suites"
+
+          puts TTY::Markdown.parse(message)
+          return
+        end
+
+        File.write(suite.short_id_file_path, suite.current_short_id_map.to_yaml)
+        puts "Short ID map saved to #{suite.short_id_file_path}"
+      end
     end
   end
 end

data/lib/inferno/apps/cli/templates/lib/%library_name%/example_suite/patient_group.rb.tt

@@ -0,0 +1,141 @@
+module <%= module_name %>
+  class PatientGroup < Inferno::TestGroup
+    title 'Patient'
+
+    description <<~DESCRIPTION
+      This scenario verifies the ability of a system to provide a Patient as described in the Example criterion.
+
+      *or*
+
+      The Example Patient sequence verifies that the system under test is able to provide correct responses for Patient queries.
+
+      ## Requirements
+
+      Patient queries must contain resources conforming to the Example Patient as specified in the Example Implementation Guide.
+
+      *or*
+
+      All Must Support elements must be seen before the test can pass, as well as Data Absent Reason to demonstrate that the server
+      can properly handle missing data. Note that Organization, Practitioner, and RelatedPerson resources must be accessible as
+      references in some Example profiles to satisfy must support requirements, and those references will be validated to their Example
+      profile. These resources will not be tested for FHIR search support.
+
+      ## <*If applicable*> Dependencies
+      Prior to running this scenario, systems must recieve a verified access token from one of the previous SMART App Launch scenarios.
+
+      *or*
+
+      Prior to running this scenario, first run the Single Patient API tests using resource-level scopes, as this scenario uses content
+      saved from that scenario as a baseline for comparison when finer-grained scopes are granted.
+
+      ## <*If applicable*> Methodology
+
+      *Only include if different from instructions included in a parent group or suite*
+
+      The test begins by searching by one or more patients, with the expectation that the Bearer token provided to the test grants
+      access to all Resources. It uses results returned from that query to generate other queries and checks that the results are
+      consistent with the provided search parameters. It then performs a read on each Resource returned and validates the response
+      against the relevant profile as currently defined in the Example Implementation Guide.
+
+      *or*
+
+      ### Searching
+
+      This test sequence will first perform each required search associated with this resource.
+      This sequence will perform searches with the following parameters:
+      - _id
+      - identifier
+      - name
+      - birthdate + name
+      - gender + name
+
+      #### Search Parameters
+
+      The first search uses the selected patient(s) from the prior launch sequence. Any subsequent searches will look for its parameter
+      values from the results of the first search. For example, the `identifier` search in the patient sequence is performed by looking
+      for an existing `Patient.identifier` from any of the resources returned in the `_id` search. If a value cannot be found this way,
+      the search is skipped.
+
+      #### Search Validation
+
+      Inferno will retrieve up to the first 20 bundle pages of the reply for Patient resources and save them for subsequent tests.
+      Each of these resources is then checked to see if it matches the searched parameters in accordance with [FHIR search guidelines](https://www.hl7.org/fhir/search.html).
+      The test will fail, for example, if a Patient search for gender=male returns a female patient.
+
+      ### Must Support
+
+      Each profile contains elements marked as "must support". This test sequence expects to see each of these elements at least once.
+      If at least one cannot be found, the test will fail. The test will look through the Patient resources found in the first test
+      for these elements.
+
+      ### Profile Validation
+
+      Each resource returned from the first search is expected to conform to the [Example Patient Profile](https://www.example.com/patient/profile).
+      Each element is checked against teminology binding and cardinality requirements.
+
+      Elements with a required binding are validated against their bound ValueSet. If the code/system in the element is not part of the
+      ValueSet, then the test will fail.
+
+      ### Reference Validation
+
+      At least one instance of each external reference in elements marked as "must support" within the resources provided by the
+      system must resolve. The test will attempt to read each reference found and will fail if no read succeeds.
+
+      ## <*If applicable*> Running the Tests
+
+      *Only include if different from instructions included in a parent group or suite*
+
+      Register Inferno as an EHR-launched application using patient-level scopes and the following URIs:
+      - Launch URI: https://inferno.healthit.gov/suites/custom/smart/launch
+      - Redirect URI: https://inferno.healthit.gov/suites/custom/smart/redirect
+
+      ## <*If top-level group for criteria*> Relevant Specifications
+
+      The following implementation specifications are relevant to this scenario:
+      - [Specification 1 v1](https://www.example.com/spec1/v1)
+      - [Specification 1 v2](https://www.example.com/spec1/v2)
+      - [Specification 2 v5](https://www.example.com/spec1/v1)
+
+    DESCRIPTION
+
+    id :patient_group
+
+    test do
+      title 'Server returns requested Patient resource from the Patient read interaction'
+      description %(
+        Verify that Patient resources can be read from the server. Expects a 200 response that includes a Patient
+        resource whose ID matches the requested patient ID.
+      )
+
+      input :patient_id,
+            title: 'Patient ID'
+
+      # Named requests can be used by other tests
+      makes_request :patient
+
+      run do
+        fhir_read(:patient, patient_id, name: :patient)
+
+        assert_response_status(200)
+        assert_resource_type(:patient)
+        assert resource.id == patient_id,
+               "Requested resource with id #{patient_id}, received resource with id #{resource.id}"
+      end
+    end
+
+    test do
+      title 'Patient resource is valid'
+      description %(
+        Verify that the Patient resource returned from the server is a valid FHIR resource.
+      )
+      # This test will use the response from the :patient request in the
+      # previous test
+      uses_request :patient
+
+      run do
+        assert_resource_type(:patient)
+        assert_valid_resource
+      end
+    end
+  end
+end