inferno_core 0.6.10 → 0.6.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ff5c612e8052b2c2a817e5ad58d27d935b8838adc2d738a8924e61ba7ecc1a0
-  data.tar.gz: 4324693af54d9c83d4eb60f2a2b85dc8012c72d372d9858bda39693d2b1de8b4
+  metadata.gz: a7e5f22313a778dd7b70ba70eae9513bda158a7c0cd6beba9a2d195b3a37f077
+  data.tar.gz: 8e7d6155fad17e00ccc48257853f4df2cc816d0965cc0b18a50515bbd26d343f
 SHA512:
-  metadata.gz: 2cbff301066bd9b295edc08790bbf8b6d658d2cd5c647a636a09c5918ff0024fc06eb566742d271bf6ea1a145745072f16ab157a674b6be3e8076a298d8110fd
-  data.tar.gz: 5a9281d55a608b9356d535c5e7fc0b9bc8afb6d23a5cc6f76192e074582d385358b81ff07dd8959286440634b08411d8866b306313c4826d81ca85eb02a4d225
+  metadata.gz: 56b26537ac370a6b475ab33895ad91e3d5cf5ff588cd3fa5be979c6e250e5aa703a76c093ea46c787ecd2c4f061f1f55e39c43ba245219618c18b9a44e522f3e
+  data.tar.gz: a1549f23c6e8c501286bc490c6821be19da1a730b950a497bdd7a2c7d61c392090e0ac1603f251e4d033abedbc4457e1dfcbf7ecac5bcfcd875392e7125ca4ef

data/lib/inferno/apps/cli/evaluate.rb

@@ -27,10 +27,11 @@ module Inferno
 
         ig_path = absolute_path_with_home_expansion(ig_path)
         data_path = absolute_path_with_home_expansion(data_path) if data_path
+        config_path = absolute_path_with_home_expansion(options[:config]) if options[:config]
 
         Dir.chdir(tmpdir) do
           Migration.new.run(Logger::FATAL) # Hide migration output for evaluator
-          evaluate(ig_path, data_path, options)
+          evaluate(ig_path, data_path, config_path, options)
         end
       ensure
         system("#{services_base_command} down #{services_names}")
@@ -57,14 +58,14 @@ module Inferno
       end
 
       # @see Inferno::CLI::Main#evaluate
-      def evaluate(ig_path, data_path, options)
+      def evaluate(ig_path, data_path, config_path, options)
         # NOTE: repositories is required here rather than at the top of the file because
         # the tree of requires means that this file and its requires get required by every CLI app.
         # Sequel::Model, used in some repositories, fetches the table schema at instantiation.
         # This breaks the `migrate` task by fetching a table before the task runs/creates it.
         require_relative '../../repositories'
 
-        validate_args(ig_path, data_path)
+        validate_args(ig_path, data_path, config_path)
         ig = Inferno::Repositories::IGs.new.find_or_load(ig_path)
 
         check_ig_version(ig)
@@ -80,17 +81,17 @@ module Inferno
 
         evaluator = Inferno::DSL::FHIREvaluation::Evaluator.new(ig, validator)
 
-        config = Inferno::DSL::FHIREvaluation::Config.new
+        config = Inferno::DSL::FHIREvaluation::Config.new(config_path)
         results = evaluator.evaluate(data, config)
         output_results(results, options[:output])
       end
 
-      def validate_args(ig_path, data_path)
+      def validate_args(ig_path, data_path, config_path)
         raise 'A path to an IG is required!' unless ig_path
 
-        return unless data_path && (!File.directory? data_path)
+        raise "Provided path '#{data_path}' is not a directory" if data_path && (!File.directory? data_path)
 
-        raise "Provided path '#{data_path}' is not a directory"
+        raise "Provided path '#{config_path}' does not exist" if config_path && (!File.exist? config_path)
       end
 
       def check_ig_version(ig)

data/lib/inferno/apps/cli/execute.rb

@@ -6,6 +6,7 @@ Dir[File.join(__dir__, 'execute', '*_outputter.rb')].each { |outputter| require
 
 module Inferno
   module CLI
+    # @private
     class Execute
       include ::Inferno::Utils::VerifyRunnable
       include ::Inferno::Utils::PersistInputs
@@ -45,7 +46,6 @@ module Inferno
         self.options = options
 
         outputter.print_start_message(self.options)
-
         load_preset_file_and_set_preset_id
 
         results = []
@@ -104,18 +104,6 @@ module Inferno
         @outputter ||= OUTPUTTERS[options[:outputter]].new
       end
 
-      def load_preset_file_and_set_preset_id
-        return unless options[:preset_file]
-        raise StandardError, 'Cannot use `--preset-id` and `--preset-file` options together' if options[:preset_id]
-
-        raise StandardError, "File #{options[:preset_file]} not found" unless File.exist? options[:preset_file]
-
-        options[:preset_id] = JSON.parse(File.read(options[:preset_file]))['id']
-        raise StandardError, "Preset #{options[:preset_file]} is missing id" if options[:preset_id].nil?
-
-        presets_repo.insert_from_file(options[:preset_file])
-      end
-
       def all_selected_groups_and_tests
         @all_selected_groups_and_tests ||= runnables_by_short_id + groups + tests
       end
@@ -123,7 +111,7 @@ module Inferno
       def run_one(runnable, test_run)
         verify_runnable(
           runnable,
-          thor_hash_to_inputs_array(inputs_and_preset),
+          all_inputs,
           test_session.suite_options
         )
 
@@ -132,15 +120,14 @@ module Inferno
         dispatch_job(test_run)
       end
 
-      def inputs_and_preset
+      def all_inputs
         if preset
-          preset_inputs = preset.inputs.to_h do |preset_input|
-            [preset_input[:name], preset_input[:value]]
-          end
+          test_sessions_repo.apply_preset(test_session, preset.id)
+          preset_inputs = session_data_repo.get_all_from_session(test_session.id)
 
-          options.fetch(:inputs, {}).reverse_merge(preset_inputs)
+          thor_hash_to_inputs_array(options.fetch(:inputs, {})) + preset_inputs.map(&:to_hash)
         else
-          options.fetch(:inputs, {})
+          thor_hash_to_inputs_array(options.fetch(:inputs, {}))
         end
       end
 
@@ -159,6 +146,15 @@ module Inferno
         @preset
       end
 
+      def load_preset_file_and_set_preset_id
+        return unless options[:preset_file]
+        raise StandardError, 'Cannot use `--preset-id` and `--preset-file` options together' if options[:preset_id]
+
+        raise StandardError, "File #{options[:preset_file]} not found" unless File.exist? options[:preset_file]
+
+        options[:preset_id] = presets_repo.insert_from_file(options[:preset_file]).id
+      end
+
       def suite
         @suite ||= Inferno::Repositories::TestSuites.new.find(options[:suite])
 
@@ -214,7 +210,7 @@ module Inferno
         {
           test_session_id: test_session.id,
           runnable_id_key(runnable) => runnable.id,
-          inputs: thor_hash_to_inputs_array(inputs_and_preset)
+          inputs: all_inputs
         }
       end
 

data/lib/inferno/apps/cli/main.rb

@@ -6,8 +6,8 @@ require_relative 'services'
 require_relative 'suite'
 require_relative 'suites'
 require_relative 'new'
-require_relative '../../version'
 require_relative 'execute'
+require_relative '../../version'
 
 module Inferno
   module CLI
@@ -34,6 +34,9 @@ module Inferno
 
         # Loads the us core ig and evaluate the data included in the IG's example folder, with results redirected to outcome.json as an OperationOutcome
         `bundle exec inferno evaluate ./uscore.tgz --output outcome.json`
+
+        # Loads the us core ig and evaluate the data included in the IG's example folder using a custom configuration file
+        `bundle exec inferno evaluate ./uscore.tgz --config ./custom_config.yml`
       LONGDESC
       # TODO: Add options below as arguments
       option :data_path,
@@ -45,6 +48,10 @@ module Inferno
              aliases: ['-o'],
              type: :string,
              desc: 'Export evaluation result to outcome.json as an OperationOutcome'
+      option :config,
+             aliases: ['-c'],
+             type: :string,
+             desc: 'Path to a custom configuration file'
       def evaluate(ig_path)
         Evaluate.new.run(ig_path, options[:data_path], options)
       end
@@ -92,6 +99,10 @@ module Inferno
 
       desc 'suites', 'List available test suites'
       def suites
+        ENV['NO_DB'] = 'true'
+
+        require_relative '../../../inferno'
+
         Suites.new.run
       end
 

data/lib/inferno/apps/cli/requirements.rb

@@ -1,3 +1,4 @@
+require_relative 'requirements_coverage_checker'
 require_relative 'requirements_exporter'
 
 module Inferno
@@ -23,6 +24,54 @@ module Inferno
         ENV['NO_DB'] = 'true'
         RequirementsExporter.new.run_check
       end
+
+      desc 'coverage [TEST_SUITE_ID]',
+           "Check whether all of a test suite's requirements are tested. If no test suite id is provided, " \
+           'all test suites in the current test kit will be checked.'
+      long_desc <<~LONGDESC
+        Check whether all of the requirements declared by a test suite are
+        tested by the tests in the test suite
+      LONGDESC
+      def coverage(test_suite_id = nil)
+        ENV['NO_DB'] = 'true'
+
+        require_relative '../../../inferno'
+
+        Inferno::Application.start(:requirements)
+
+        if test_suite_id.present?
+          RequirementsCoverageChecker.new(test_suite_id).run
+        else
+          Inferno::Repositories::TestSuites.all.each do |test_suite|
+            if Object.const_source_location(test_suite.to_s).first.start_with? Dir.pwd
+              RequirementsCoverageChecker.new(test_suite.id).run
+            end
+          end
+        end
+      end
+
+      desc 'check_coverage [TEST_SUITE_ID]',
+           'Check whether the coverage CSV files are up to date'
+      long_desc <<~LONGDESC
+        Check whether the coverage CSV files are up to date
+      LONGDESC
+      def check_coverage(test_suite_id = nil)
+        ENV['NO_DB'] = 'true'
+
+        require_relative '../../../inferno'
+
+        Inferno::Application.start(:requirements)
+
+        if test_suite_id.present?
+          RequirementsCoverageChecker.new(test_suite_id).run_check
+        else
+          Inferno::Repositories::TestSuites.all.each do |test_suite|
+            if Object.const_source_location(test_suite.to_s).first.start_with? Dir.pwd
+              RequirementsCoverageChecker.new(test_suite.id).run_check
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/inferno/apps/cli/requirements_coverage_checker.rb

@@ -0,0 +1,238 @@
+require_relative 'requirements_exporter'
+
+module Inferno
+  module CLI
+    class RequirementsCoverageChecker
+      attr_accessor :test_suite_id, :test_suite
+
+      def initialize(test_suite_id)
+        self.test_suite_id = test_suite_id
+        self.test_suite = Inferno::Repositories::TestSuites.new.find(test_suite_id)
+      end
+
+      def short_id_header
+        "#{test_suite.title} Short ID(s)"
+      end
+
+      def full_id_header
+        "#{test_suite.title} Full ID(s)"
+      end
+
+      def output_headers
+        [
+          *(RequirementsExporter::REQUIREMENTS_OUTPUT_HEADERS - ['Sub-Requirement(s)']),
+          short_id_header,
+          full_id_header
+        ]
+      end
+
+      def test_kit_name
+        local_test_kit_gem.name
+      end
+
+      def base_requirements_folder
+        RequirementsExporter.new.base_requirements_folder
+      end
+
+      def output_folder
+        @output_folder ||= File.join(base_requirements_folder, 'generated')
+      end
+
+      def output_file_name
+        "#{test_suite_id}_requirements_coverage.csv"
+      end
+
+      def output_file_path
+        @output_file_path ||= File.join(output_folder, output_file_name)
+      end
+
+      def suite_requirements
+        @suite_requirements ||=
+          Inferno::Repositories::Requirements.new.requirements_for_suite(test_suite_id)
+      end
+
+      def tested_requirement_ids
+        @tested_requirement_ids ||= test_suite.all_verified_requirements
+      end
+
+      def suite_runnables
+        @suite_runnables ||= [test_suite] + test_suite.all_descendants
+      end
+
+      def untested_requirements
+        @untested_requirements ||= []
+      end
+
+      def new_csv # rubocop:disable Metrics/CyclomaticComplexity
+        @new_csv ||=
+          CSV.generate(+"\xEF\xBB\xBF") do |csv|
+            csv << output_headers
+
+            suite_requirements.each do |requirement|
+              if requirement.not_tested_reason.present?
+                long_ids = 'NA'
+                short_ids = 'NA'
+              else
+                runnables_for_requirement =
+                  suite_runnables.select { |runnable| runnable.verifies_requirements.include? requirement.id }
+                long_ids = runnables_for_requirement&.map(&:id)&.join(', ')
+                short_ids =
+                  runnables_for_requirement
+                    &.map { |runnable| runnable < Inferno::Entities::TestSuite ? 'suite' : runnable.short_id }
+                    &.join(', ')
+              end
+
+              untested_requirements << runnables_for_requirement if runnables_for_requirement.blank?
+
+              row = [
+                requirement.requirement_set,
+                requirement.id.delete_prefix("#{requirement.requirement_set}@"),
+                requirement.url,
+                requirement.requirement,
+                requirement.conformance,
+                requirement.actor,
+                requirement.conditionality,
+                requirement.not_tested_reason,
+                requirement.not_tested_details,
+                short_ids,
+                long_ids
+              ]
+
+              csv << row
+            end
+          end
+      end
+
+      def input_requirement_ids
+        @input_requirement_ids ||= input_rows.map { |row| "#{row['Req Set']}@#{row['ID']}" }
+      end
+
+      # The requirements present in Inferno that aren't in the input spreadsheet
+      def unmatched_requirement_ids
+        @unmatched_requirement_ids ||=
+          tested_requirement_ids - suite_requirements.map(&:id)
+      end
+
+      def unmatched_requirement_rows
+        unmatched_requirement_ids.flat_map do |requirement_id|
+          runnables_for_requirement =
+            suite_runnables.select { |runnable| runnable.verifies_requirements.include? requirement_id }
+
+          runnables_for_requirement.map do |runnable|
+            [requirement_id, runnable.short_id, runnable.id]
+          end
+        end
+      end
+
+      def old_csv
+        @old_csv ||= File.read(output_file_path)
+      end
+
+      def run
+        unless test_suite.present?
+          puts "Could not find test suite: #{test_suite_id}. Aborting requirements coverage generation..."
+          exit(1)
+        end
+
+        puts
+
+        if unmatched_requirement_ids.present?
+          puts "WARNING: The following requirements indicated in the test suite #{test_suite_id} are not present in " \
+               "the suite's requirement sets:"
+          output_requirements_map_table(unmatched_requirement_rows)
+        end
+
+        if File.exist?(output_file_path)
+          if old_csv == new_csv
+            puts "'#{output_file_name}' file is up to date."
+            return
+          else
+            puts 'Requirements coverage has changed.'
+          end
+        else
+          puts "No existing #{output_file_name}."
+        end
+
+        puts "Writing to file #{output_file_path}..."
+        FileUtils.mkdir_p(output_folder)
+        File.write(output_file_path, new_csv)
+        puts 'Done.'
+      end
+
+      def run_check
+        unless test_suite.present?
+          puts "Could not find test suite: #{test_suite_id}. Aborting requirements coverage generation..."
+          exit(1)
+        end
+
+        puts
+
+        if unmatched_requirement_ids.any?
+          puts "WARNING: The following requirements indicated in the test suite #{test_suite_id} are not present in " \
+               "the suite's requirement sets:"
+          output_requirements_map_table(unmatched_requirement_rows)
+        end
+
+        if File.exist?(output_file_path)
+          if old_csv == new_csv
+            puts "'#{output_file_name}' file is up to date."
+            return unless unmatched_requirement_ids.present?
+          else
+            puts <<~MESSAGE
+              #{output_file_name} file is out of date.
+              To regenerate the file, run:
+
+                  bundle exec inferno requirements coverage
+
+            MESSAGE
+          end
+        else
+          puts <<~MESSAGE
+            No existing #{output_file_name} file.
+            To generate the file, run:
+
+                  bundle exec inferno requirements coverage
+
+          MESSAGE
+        end
+
+        puts 'Check failed.'
+        exit(1)
+      end
+
+      # Output the requirements in the map like so:
+      #
+      # requirement_id | short_id   | full_id
+      # ---------------+------------+----------
+      # req-id-1       | short-id-1 | full-id-1
+      # req-id-2       | short-id-2 | full-id-2
+      #
+      def output_requirements_map_table(requirements_rows)
+        headers = %w[requirement_id short_id full_id]
+        col_widths = headers.map(&:length)
+        col_widths[0] = [col_widths[0], *requirements_rows.map { |row| row[0].length }].max
+        col_widths[1] = [col_widths[1], *requirements_rows.map { |row| row[1].length }].max
+        col_widths[2] = [col_widths[2], *requirements_rows.map { |row| row[2].length }].max
+        col_widths.map! { |width| width + 3 }
+
+        puts [
+          headers[0].ljust(col_widths[0]),
+          headers[1].ljust(col_widths[1]),
+          headers[2].ljust(col_widths[2])
+        ].join(' | ')
+        puts col_widths.map { |width| '-' * width }.join('-+-')
+        output_requirements_map_table_contents(requirements_rows, col_widths)
+      end
+
+      def output_requirements_map_table_contents(requirements_rows, col_widths)
+        requirements_rows.each do |requirements_row|
+          puts [
+            requirements_row[0].ljust(col_widths[0]),
+            requirements_row[1].ljust(col_widths[1]),
+            requirements_row[2].ljust(col_widths[2])
+          ].join(' | ')
+        end
+      end
+    end
+  end
+end

data/lib/inferno/apps/cli/suite.rb

@@ -15,6 +15,9 @@ module Inferno
       option :filename, banner: '<filename>', aliases: [:f]
       def input_template(suite_id)
         ENV['NO_DB'] = 'true'
+
+        require_relative '../../../inferno'
+
         SuiteInputTemplate.new.run(suite_id, options)
       end
 
@@ -24,6 +27,9 @@ module Inferno
       LONGDESC
       def describe(suite_id)
         ENV['NO_DB'] = 'true'
+
+        require_relative '../../../inferno'
+
         Inferno::Application.start(:suites)
 
         suite = Inferno::Repositories::TestSuites.new.find(suite_id)
@@ -59,6 +65,9 @@ module Inferno
       LONGDESC
       def lock_short_ids(suite_id)
         ENV['NO_DB'] = 'true'
+
+        require_relative '../../../inferno'
+
         Inferno::Application.start(:suites)
 
         suite = Inferno::Repositories::TestSuites.new.find(suite_id)

data/lib/inferno/apps/web/controllers/test_sessions/results/io_value.rb

@@ -0,0 +1,68 @@
+module Inferno
+  module Web
+    module Controllers
+      module TestSessions
+        module Results
+          class IoValue < Controller
+            include Import[
+              test_sessions_repo: 'inferno.repositories.test_sessions',
+              results_repo: 'inferno.repositories.results'
+            ]
+
+            def handle(req, res)
+              test_session_id = req.params[:id]
+              result_id = req.params[:result_id]
+              type = req.params[:type]
+              name = req.params[:name]
+
+              unless %w[inputs outputs].include?(type)
+                res.status = 400
+                res.body = { error: 'Invalid I/O type. Must be "inputs" or "outputs".' }.to_json
+                return
+              end
+
+              test_session_results = results_repo.current_results_for_test_session(test_session_id)
+              result = test_session_results.find { |r| r.id == result_id }
+
+              if result.nil?
+                res.status = 404
+                res.body = { error: "Result '#{result_id}' not found for test session '#{test_session_id}'." }.to_json
+                return
+              end
+
+              entry = result_io_by_name(result, type, name)
+              value = entry&.dig('value')
+
+              if value.blank?
+                res.status = 404
+                res.body = { error: "#{type.singularize.capitalize} '#{name}' not found or missing value." }.to_json
+                return
+              end
+
+              res.body = value.is_a?(Hash) ? value.to_json : value
+              res.content_type = content_type_for(value)
+            end
+
+            private
+
+            def result_io_by_name(result, io_type, name)
+              result.public_send(io_type)
+                .find { |item| item['name'] == name }
+            end
+
+            def content_type_for(value)
+              return 'application/json' if value.is_a?(Hash)
+
+              trimmed = value&.strip
+
+              return 'application/json' if trimmed&.start_with?('{', '[')
+              return 'application/xml' if trimmed&.start_with?('<') && trimmed.end_with?('>')
+
+              'text/plain'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inferno/apps/web/router.rb

@@ -1,4 +1,5 @@
 require 'erb'
+require_relative '../../feature'
 
 Dir.glob(File.join(__dir__, 'controllers', '**', '*.rb')).each { |path| require_relative path }
 
@@ -33,6 +34,9 @@ module Inferno
           get '/:id/results',
               to: Inferno::Web::Controllers::TestSessions::Results::Index,
               as: :results
+          get '/:id/results/:result_id/io/:type/:name',
+              to: Inferno::Web::Controllers::TestSessions::Results::IoValue,
+              as: :result_io_value
           get '/:id/session_data',
               to: Inferno::Web::Controllers::TestSessions::SessionData::Index
           put '/:id/session_data/apply_preset',
@@ -47,13 +51,17 @@ module Inferno
           put '/:id/check_configuration',
               to: Inferno::Web::Controllers::TestSuites::CheckConfiguration,
               as: :check_configuration
-          get ':id/requirements',
-              to: Inferno::Web::Controllers::TestSuites::Requirements::Index,
-              as: :requirements
+          if Feature.requirements_enabled?
+            get ':id/requirements',
+                to: Inferno::Web::Controllers::TestSuites::Requirements::Index,
+                as: :requirements
+          end
         end
 
-        scope 'requirements' do
-          get '/:id', to: Inferno::Web::Controllers::Requirements::Show, as: :show
+        if Feature.requirements_enabled?
+          scope 'requirements' do
+            get '/:id', to: Inferno::Web::Controllers::Requirements::Show, as: :show
+          end
         end
 
         get '/requests/:id', to: Inferno::Web::Controllers::Requests::Show, as: :requests_show

data/lib/inferno/apps/web/serializers/result.rb

@@ -21,11 +21,11 @@ module Inferno
         field :optional?, name: :optional
 
         field :inputs do |result, _options|
-          result.input_json.present? ? JSON.parse(result.input_json) : []
+          result.handle_large_io('inputs')
         end
 
         field :outputs do |result, _options|
-          result.output_json.present? ? JSON.parse(result.output_json) : []
+          result.handle_large_io('outputs')
         end
 
         association :messages, blueprint: Message, if: :field_present?

data/lib/inferno/apps/web/serializers/serializer.rb

@@ -1,5 +1,6 @@
 require 'oj'
 require 'blueprinter'
+require_relative '../../../feature'
 
 module Inferno
   module Web
@@ -13,6 +14,12 @@ module Inferno
             result.send(name).present?
           end
         end
+
+        # When removing the feature flag, replace all instances of this method
+        # with `.field_present?`
+        def self.field_present_and_requirements_enabled?(field_name, result, options)
+          field_present?(field_name, result, options) && Feature.requirements_enabled?
+        end
       end
     end
   end

data/lib/inferno/apps/web/serializers/test.rb

@@ -20,7 +20,7 @@ module Inferno
         field :input_instructions
         field :user_runnable?, name: :user_runnable
         field :optional?, name: :optional
-        field :verifies_requirements, if: :field_present?
+        field :verifies_requirements, if: :field_present_and_requirements_enabled?
       end
     end
   end

data/lib/inferno/apps/web/serializers/test_group.rb

@@ -32,7 +32,7 @@ module Inferno
           Input.render_as_hash(group.available_inputs(suite_options).values)
         end
         field :output_definitions, name: :outputs, extractor: HashValueExtractor
-        field :verifies_requirements, if: :field_present?
+        field :verifies_requirements, if: :field_present_and_requirements_enabled?
       end
     end
   end