davinci_dtr_test_kit 0.14.2 → 0.15.0

data/lib/davinci_dtr_test_kit/urls.rb

@@ -2,6 +2,7 @@
 
 module DaVinciDTRTestKit
   FHIR_BASE_PATH = '/fhir'
+  METADATA_PATH = "#{FHIR_BASE_PATH}/metadata".freeze
   SMART_CONFIG_PATH = "#{FHIR_BASE_PATH}/.well-known/smart-configuration".freeze
   OPENID_CONFIG_PATH = "#{FHIR_BASE_PATH}/.well-known/openid-configuration".freeze
   JKWS_PATH = "#{FHIR_BASE_PATH}/.well-known/jwks.json".freeze

data/lib/davinci_dtr_test_kit/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 module DaVinciDTRTestKit
-  VERSION = '0.14.2'
+  VERSION = '0.15.0'
+  LAST_UPDATED = '2025-02-25'
 end

data/lib/davinci_dtr_test_kit.rb

@@ -1,3 +1,5 @@
+require_relative 'davinci_dtr_test_kit/metadata'
+require_relative 'inferno_requirements_tools/ext/inferno_core/runnable'
 require_relative 'davinci_dtr_test_kit/dtr_payer_server_suite'
 require_relative 'davinci_dtr_test_kit/dtr_smart_app_suite'
 require_relative 'davinci_dtr_test_kit/dtr_full_ehr_suite'

data/lib/inferno_requirements_tools/ext/inferno_core/runnable.rb

@@ -0,0 +1,22 @@
+module Inferno
+  module DSL
+    # This module contains the DSL for defining child entities in the test
+    # definition framework.
+    module Runnable
+      # Set/Get the IDs of requirements verifed by this runnable
+      # Set with [] to clear the list
+      #
+      # @param requirements [Array<String>]
+      # @return [Array<String>] the requirement IDs
+      def verifies_requirements(*requirement_ids)
+        if requirement_ids.empty?
+          @requirement_ids || []
+        elsif requirement_ids == [[]]
+          @requirement_ids = []
+        else
+          @requirement_ids = requirement_ids
+        end
+      end
+    end
+  end
+end

data/lib/inferno_requirements_tools/rake/rakefile_template

@@ -0,0 +1,31 @@
+namespace :requirements do
+  desc 'Generate requirements coverage CSV'
+  task :generate_coverage do
+    require 'inferno'
+    Inferno::Application.start(:suites)
+
+    require_relative 'lib/inferno_requirements_tools/tasks/requirements_coverage'
+    InfernoRequirementsTools::Tasks::RequirementsCoverage.new.run
+  end
+
+  desc 'Check if requirements coverage CSV is up-to-date'
+  task :check_coverage do
+    require 'inferno'
+    Inferno::Application.start(:suites)
+
+    require_relative 'lib/inferno_requirements_tools/tasks/requirements_coverage'
+    InfernoRequirementsTools::Tasks::RequirementsCoverage.new.run_check
+  end
+
+  desc 'Collect requirements and planned not tested requirements into CSVs'
+  task :collect, [:requirements_dir, :test_kit_dir] => [] do |_t, args|
+    require_relative 'lib/inferno_requirements_tools/tasks/collect_requirements'
+    InfernoRequirementsTools::Tasks::CollectRequirements.run(args.requirements_dir, args.test_kit_dir)
+  end
+
+  desc 'Check if requirements and planned not tested CSVs are up-to-date'
+  task :check_collection, [:requirements_dir, :test_kit_dir] => [] do |_t, args|
+    require_relative 'lib/inferno_requirements_tools/tasks/collect_requirements'
+    InfernoRequirementsTools::Tasks::CollectRequirements.check(args.requirements_dir, args.test_kit_dir)
+  end
+end

data/lib/inferno_requirements_tools/tasks/requirements_coverage.rb

@@ -0,0 +1,284 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'yaml'
+require_relative '../ext/inferno_core/runnable'
+
+module InfernoRequirementsTools
+  module Tasks
+    # This class manages the mapping of test kit tests to requirements that they verify
+    # and creates a CSV file with the tests that cover each requirement.
+    # It expects a CSV file in the repo at `lib/[test kit id]/requirements/[test kit id]_requirements.csv`
+    # that serves as the source of the requirement set for the test kit. The requirements in
+    # this files are identified by a requirement set and an id and tests, groups, and suites
+    # within in the test kit can claim that they verify a requirement by including a reference
+    # to that requirementin the form <requirement set>@<id> in their `verifies_requirements` field.
+    # Requirements that are out of scope can be listed in a companion file
+    # `lib/[test kit id]/requirements/[test kit id]_out_of_scope_requirements.csv`.
+    #
+    # The `run` method generates a CSV file at
+    # `lib/[test kit id]/requirements/generated/[test kit id]_requirements_coverage.csv``.
+    # This file will be identical to the input spreadsheet, plus an additional column which holds a comma separated
+    # list of inferno test IDs that test each requirement. These test IDs are Inferno short form IDs that represent the
+    # position of the test within its group and suite. For example, the fifth test in the second group will have an ID
+    # of 2.05. This ID is also shown in the Inferno web UI.
+    #
+    # The `run_check` method will check whether the previously generated file is up-to-date.
+    class RequirementsCoverage
+      VERSION = '0.2.0' # update when making meaningful changes to this method for tracking used versions
+      CONFIG = YAML.load_file(File.join('lib', 'requirements_config.yaml'))
+
+      TEST_KIT_ID = CONFIG['test_kit_id']
+      TEST_SUITES = CONFIG['suites'].map do |test_suite|
+        Object.const_get(test_suite['class_name'])
+      end
+
+      SUITE_ID_TO_ACTOR_MAP = CONFIG['suites'].each_with_object({}) do |test_suite, hash|
+        hash[test_suite['id']] = test_suite['suite_actor']
+      end
+
+      # Derivative constants
+      TEST_KIT_CODE_FOLDER = TEST_KIT_ID
+      DASHERIZED_TEST_KIT_ID = TEST_KIT_ID.gsub('_', '-')
+      INPUT_HEADERS = [
+        'Req Set',
+        'ID',
+        'URL',
+        'Requirement',
+        'Conformance',
+        'Actor',
+        'Sub-Requirement(s)',
+        'Conditionality'
+      ].freeze
+      SHORT_ID_HEADER = 'Short ID(s)'
+      FULL_ID_HEADER = 'Full ID(s)'
+      INPUT_FILE_NAME = "#{DASHERIZED_TEST_KIT_ID}_requirements.csv".freeze
+      INPUT_FILE = File.join('lib', TEST_KIT_CODE_FOLDER, 'requirements', INPUT_FILE_NAME).freeze
+      NOT_TESTED_FILE_NAME = "#{DASHERIZED_TEST_KIT_ID}_out_of_scope_requirements.csv".freeze
+      NOT_TESTED_FILE = File.join('lib', TEST_KIT_CODE_FOLDER, 'requirements', NOT_TESTED_FILE_NAME).freeze
+      OUTPUT_HEADERS = INPUT_HEADERS + TEST_SUITES.flat_map do |suite|
+                                         ["#{suite.title} #{SHORT_ID_HEADER}", "#{suite.title} #{FULL_ID_HEADER}"]
+                                       end
+      OUTPUT_FILE_NAME = "#{DASHERIZED_TEST_KIT_ID}_requirements_coverage.csv".freeze
+      OUTPUT_FILE_DIRECTORY = File.join('lib', TEST_KIT_CODE_FOLDER, 'requirements', 'generated')
+      OUTPUT_FILE = File.join(OUTPUT_FILE_DIRECTORY, OUTPUT_FILE_NAME).freeze
+
+      def input_rows
+        @input_rows ||=
+          CSV.parse(File.open(INPUT_FILE, 'r:bom|utf-8'), headers: true).map do |row|
+            row.to_h.slice(*INPUT_HEADERS)
+          end
+      end
+
+      def not_tested_requirements_map
+        @not_tested_requirements_map ||= load_not_tested_requirements
+      end
+
+      def load_not_tested_requirements
+        return {} unless File.exist?(NOT_TESTED_FILE)
+
+        not_tested_requirements = {}
+        CSV.parse(File.open(NOT_TESTED_FILE, 'r:bom|utf-8'), headers: true).each do |row|
+          row_hash = row.to_h
+          not_tested_requirements["#{row_hash['Req Set']}@#{row_hash['ID']}"] = row_hash
+        end
+
+        not_tested_requirements
+      end
+
+      # Of the form:
+      # {
+      #     'req-id-1': [
+      #       { short_id: 'short-id-1', full_id: 'long-id-1', suite_id: 'suite-id-1' },
+      #       { short_id: 'short-id-2', full_id: 'long-id-2', suite_id: 'suite-id-2' }
+      #     ],
+      #     'req-id-2': [{ short_id: 'short-id-3', full_id: 'long-id-3', suite_id: 'suite-id-3' }],
+      #     ...
+      # }
+      def inferno_requirements_map
+        @inferno_requirements_map ||= TEST_SUITES.each_with_object({}) do |suite, requirements_map|
+          serialize_requirements(suite, 'suite', suite.id, requirements_map)
+          suite.groups.each do |group|
+            map_group_requirements(group, suite.id, requirements_map)
+          end
+        end
+      end
+
+      def new_csv
+        @new_csv ||=
+          CSV.generate(+"\xEF\xBB\xBF") do |csv|
+            csv << OUTPUT_HEADERS
+            input_rows.each do |row| # NOTE: use row order from source file
+              next if row['Conformance'] == 'DEPRECATED' # filter out deprecated rows
+
+              TEST_SUITES.each do |suite|
+                suite_actor = SUITE_ID_TO_ACTOR_MAP[suite.id]
+                if row['Actor']&.include?(suite_actor)
+                  add_suite_tests_for_row(row, suite)
+                else
+                  row["#{suite.title} #{SHORT_ID_HEADER}"] = 'NA'
+                  row["#{suite.title} #{FULL_ID_HEADER}"] = 'NA'
+                end
+              end
+              csv << row.values
+            end
+          end
+      end
+
+      def add_suite_tests_for_row(row, suite)
+        set_and_req_id = "#{row['Req Set']}@#{row['ID']}"
+        items = get_items_for_requirement(set_and_req_id, suite)
+        short_ids = items[0]
+        full_ids = items[1]
+        if short_ids.blank? && not_tested_requirements_map.key?(set_and_req_id)
+          row["#{suite.title} #{SHORT_ID_HEADER}"] = 'Not Tested'
+          row["#{suite.title} #{FULL_ID_HEADER}"] = 'Not Tested'
+        else
+          row["#{suite.title} #{SHORT_ID_HEADER}"] = short_ids&.join(', ')
+          row["#{suite.title} #{FULL_ID_HEADER}"] = full_ids&.join(', ')
+        end
+      end
+
+      def get_items_for_requirement(set_and_req_id, suite)
+        suite_requirement_items = inferno_requirements_map[set_and_req_id]&.filter do |item|
+          item[:suite_id] == suite.id
+        end
+        [
+          suite_requirement_items&.map { |item| item[:short_id] },
+          suite_requirement_items&.map { |item| item[:full_id] }
+        ]
+      end
+
+      def input_requirement_ids
+        @input_requirement_ids ||= input_rows.map { |row| "#{row['Req Set']}@#{row['ID']}" }
+      end
+
+      # The requirements present in Inferno that aren't in the input spreadsheet
+      def unmatched_requirements_map
+        @unmatched_requirements_map ||= inferno_requirements_map.except(*input_requirement_ids)
+      end
+
+      def old_csv
+        @old_csv ||= File.read(OUTPUT_FILE)
+      end
+
+      def run
+        unless File.exist?(INPUT_FILE)
+          puts "Could not find input file: #{INPUT_FILE}. Aborting requirements coverage generation..."
+          exit(1)
+        end
+
+        if unmatched_requirements_map.any?
+          puts "WARNING: The following requirements indicated in the test kit are not present in #{INPUT_FILE_NAME}"
+          output_requirements_map_table(unmatched_requirements_map)
+        end
+
+        if File.exist?(OUTPUT_FILE)
+          if old_csv == new_csv
+            puts "'#{OUTPUT_FILE_NAME}' file is up to date."
+            return
+          else
+            puts 'Requirements coverage has changed.'
+          end
+        else
+          puts "No existing #{OUTPUT_FILE_NAME}."
+        end
+
+        puts "Writing to file #{OUTPUT_FILE}..."
+        FileUtils.mkdir_p(OUTPUT_FILE_DIRECTORY)
+        File.write(OUTPUT_FILE, new_csv)
+        puts 'Done.'
+      end
+
+      def run_check
+        unless File.exist?(INPUT_FILE)
+          puts "Could not find input file: #{INPUT_FILE}. Aborting requirements coverage check..."
+          exit(1)
+        end
+
+        if unmatched_requirements_map.any?
+          puts "The following requirements indicated in the test kit are not present in #{INPUT_FILE_NAME}"
+          output_requirements_map_table(unmatched_requirements_map)
+        end
+
+        if File.exist?(OUTPUT_FILE)
+          if old_csv == new_csv
+            puts "'#{OUTPUT_FILE_NAME}' file is up to date."
+            return unless unmatched_requirements_map.any?
+          else
+            puts <<~MESSAGE
+              #{OUTPUT_FILE_NAME} file is out of date.
+              To regenerate the file, run:
+
+                  bundle exec rake requirements:generate_coverage
+
+            MESSAGE
+          end
+        else
+          puts <<~MESSAGE
+            No existing #{OUTPUT_FILE_NAME} file.
+            To generate the file, run:
+
+                  bundle exec rake requirements:generate_coverage
+
+          MESSAGE
+        end
+
+        puts 'Check failed.'
+        exit(1)
+      end
+
+      def map_group_requirements(group, suite_id, requirements_map)
+        serialize_requirements(group, group.short_id, suite_id, requirements_map)
+        group.tests&.each { |test| serialize_requirements(test, test.short_id, suite_id, requirements_map) }
+        group.groups&.each { |subgroup| map_group_requirements(subgroup, suite_id, requirements_map) }
+      end
+
+      def serialize_requirements(runnable, short_id, suite_id, requirements_map)
+        runnable.verifies_requirements&.each do |requirement_id|
+          requirement_id_string = requirement_id.to_s
+
+          requirements_map[requirement_id_string] ||= []
+          requirements_map[requirement_id_string] << { short_id:, full_id: runnable.id, suite_id: }
+        end
+      end
+
+      # Output the requirements in the map like so:
+      #
+      # requirement_id | short_id   | full_id
+      # ---------------+------------+----------
+      # req-id-1       | short-id-1 | full-id-1
+      # req-id-2       | short-id-2 | full-id-2
+      #
+      def output_requirements_map_table(requirements_map)
+        headers = %w[requirement_id short_id full_id]
+        col_widths = headers.map(&:length)
+        col_widths[0] = [col_widths[0], requirements_map.keys.map(&:length).max].max
+        col_widths[1] = ([col_widths[1]] + requirements_map.values.flatten.map { |item| item[:short_id].length }).max
+        col_widths[2] = ([col_widths[2]] + requirements_map.values.flatten.map { |item| item[:full_id].length }).max
+        col_widths.map { |width| width + 3 }
+
+        puts [
+          headers[0].ljust(col_widths[0]),
+          headers[1].ljust(col_widths[1]),
+          headers[2].ljust(col_widths[2])
+        ].join(' | ')
+        puts col_widths.map { |width| '-' * width }.join('-+-')
+        output_requirements_map_table_contents(requirements_map, col_widths)
+        puts
+      end
+
+      def output_requirements_map_table_contents(requirements_map, col_widths)
+        requirements_map.each do |requirement_id, runnables|
+          runnables.each do |runnable|
+            puts [
+              requirement_id.ljust(col_widths[0]),
+              runnable[:short_id].ljust(col_widths[1]),
+              runnable[:full_id].ljust(col_widths[2])
+            ].join(' | ')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/requirements_config.yaml

@@ -0,0 +1,25 @@
+test_kit_id: davinci_dtr_test_kit
+
+suites: 
+  - id: dtr_full_ehr
+    class_name: DaVinciDTRTestKit::DTRFullEHRSuite
+    suite_actor: Full EHR
+  - id: dtr_smart_app
+    class_name: DaVinciDTRTestKit::DTRSmartAppSuite
+    suite_actor: SMART Client
+  - id: dtr_light_ehr
+    class_name: DaVinciDTRTestKit::DTRLightEHRSuite
+    suite_actor: Light EHR
+  - id: dtr_payer_server
+    class_name: DaVinciDTRTestKit::DTRPayerServerSuite
+    suite_actor: Payer Service
+
+requirement_sets:
+  - id: hl7.fhir.us.davinci-dtr_2.0.1
+    folder: Documentation Templates and Rules (DTR)
+    requirements: All
+    actor_map:
+      - {spec: Full EHR, test_kit: Full EHR}
+      - {spec: Light EHR, test_kit: Light EHR}
+      - {spec: SMART Client, test_kit: SMART Client}
+      - {spec: Payer Service, test_kit: Payer Service}