davinci_pdex_test_kit 0.11.0 → 0.12.1

data/lib/inferno_requirements_tools/tasks/collect_requirements.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+require 'CSV'
+require 'roo'
+require 'yaml'
+require 'csv'
+
+module InfernoRequirementsTools
+  module Tasks
+    # This class manages the collection of requirements details from
+    # requirements planning excel workbooks into a CSV representation.
+    # Currently splits out Requirements and Planned Not Tested Requirements
+    # into two separate files.
+    #
+    # The `run_check` method will check whether the previously generated file is up-to-date.
+    class CollectRequirements
+      CONFIG = YAML.load_file(File.join('lib', 'requirements_config.yaml'))
+
+      TEST_KIT_ID = CONFIG['test_kit_id']
+      INPUT_SETS = CONFIG['requirement_sets'].map do |req_set|
+        req_set['id']
+      end
+
+      # Derivative constants
+      TEST_KIT_CODE_FOLDER = TEST_KIT_ID.gsub('-', '_')
+      DASHERIZED_TEST_KIT_ID = TEST_KIT_ID.gsub('_', '-')
+      INPUT_HEADERS =
+        [
+          'ID*',
+          'URL*',
+          'Requirement*',
+          'Conformance*',
+          'Actor*',
+          'Sub-Requirement(s)',
+          'Conditionality',
+          'Verifiable?',
+          'Verifiability Details',
+          'Planning To Test?',
+          'Planning To Test Details'
+        ].freeze
+      REQUIREMENTS_OUTPUT_HEADERS =
+        [
+          'Req Set',
+          'ID',
+          'URL',
+          'Requirement',
+          'Conformance',
+          'Actor',
+          'Sub-Requirement(s)',
+          'Conditionality'
+        ].freeze
+      REQUIREMENTS_OUTPUT_FILE_NAME = "#{DASHERIZED_TEST_KIT_ID}_requirements.csv".freeze
+      REQUIREMENTS_OUTPUT_FILE =
+        File.join('lib', TEST_KIT_CODE_FOLDER, 'requirements', REQUIREMENTS_OUTPUT_FILE_NAME).freeze
+      PLANNED_NOT_TESTED_OUTPUT_HEADERS = ['Req Set', 'ID', 'Reason', 'Details'].freeze
+      PLANNED_NOT_TESTED_OUTPUT_FILE_NAME = "#{DASHERIZED_TEST_KIT_ID}_out_of_scope_requirements.csv".freeze
+      PLANNED_NOT_TESTED_OUTPUT_FILE =
+        File.join('lib', TEST_KIT_CODE_FOLDER, 'requirements', PLANNED_NOT_TESTED_OUTPUT_FILE_NAME).freeze
+
+      def available_input_worksheets
+        @available_input_worksheets ||= Dir.glob(File.join(@input_directory, '*.xlsx')).reject { |f| f.include?('~$') }
+      end
+
+      # Of the form:
+      # {
+      #   req_set_id_1: [row1, row2, row 3, ...],
+      #   req_set_id_2: [row1, row2, row 3, ...]
+      # }
+      def input_requirement_sets
+        @input_requirement_sets ||= INPUT_SETS.each_with_object({}) do |req_set_id, hash|
+          req_set_file = available_input_worksheets.find { |worksheet_file| worksheet_file.include?(req_set_id) }
+
+          hash[req_set_id] =
+            unless req_set_file.nil?
+              CSV.parse(Roo::Spreadsheet.open(req_set_file).sheet('Requirements').to_csv,
+                        headers: true).map do |row|
+                row.to_h.slice(*INPUT_HEADERS)
+              end
+            end
+        end
+      end
+
+      def new_requirements_csv
+        @new_requirements_csv ||=
+          CSV.generate(+"\xEF\xBB\xBF") do |csv| # start with an unnecessary BOM to make viewing in excel easier
+            csv << REQUIREMENTS_OUTPUT_HEADERS
+
+            input_requirement_sets.each do |req_set_id, input_rows|
+              input_rows.each do |input_row| # NOTE: use row order from source file
+                csv << REQUIREMENTS_OUTPUT_HEADERS.map do |header|
+                  header == 'Req Set' ? req_set_id : input_row[header] || input_row["#{header}*"]
+                end
+              end
+            end
+          end
+      end
+
+      def old_requirements_csv
+        @old_requirements_csv ||= File.read(REQUIREMENTS_OUTPUT_FILE)
+      end
+
+      def new_planned_not_tested_csv
+        @new_planned_not_tested_csv ||=
+          CSV.generate(+"\xEF\xBB\xBF") do |csv| # start with an unnecessary BOM to make viewing in excel easier
+            csv << PLANNED_NOT_TESTED_OUTPUT_HEADERS
+
+            input_requirement_sets.each do |req_set_id, input_rows|
+              input_rows.each do |row|
+                if spreadsheet_value_falsy?(row['Verifiable?'])
+                  csv << [req_set_id, row['ID*'], 'Not Verifiable', row['Verifiability Details']]
+                elsif spreadsheet_value_falsy?(row['Planning To Test?'])
+                  csv << [req_set_id, row['ID*'], 'Not Tested', row['Planning To Test Details']]
+                end
+              end
+            end
+          end
+      end
+
+      def old_planned_not_tested_csv
+        @old_planned_not_tested_csv ||= File.read(PLANNED_NOT_TESTED_OUTPUT_FILE)
+      end
+
+      def run(input_directory)
+        @input_directory = input_directory
+        check_presence_of_input_files
+
+        update_requirements =
+          if File.exist?(REQUIREMENTS_OUTPUT_FILE)
+            if old_requirements_csv == new_requirements_csv
+              puts "'#{REQUIREMENTS_OUTPUT_FILE_NAME}' file is up to date."
+              false
+            else
+              puts 'Requirements set has changed.'
+              true
+            end
+          else
+            puts "No existing #{REQUIREMENTS_OUTPUT_FILE_NAME}."
+            true
+          end
+
+        if update_requirements
+          puts "Writing to file #{REQUIREMENTS_OUTPUT_FILE}..."
+          File.write(REQUIREMENTS_OUTPUT_FILE, new_requirements_csv, encoding: Encoding::UTF_8)
+        end
+
+        udpate_planned_not_tested =
+          if File.exist?(PLANNED_NOT_TESTED_OUTPUT_FILE)
+            if old_planned_not_tested_csv == new_planned_not_tested_csv
+              puts "'#{PLANNED_NOT_TESTED_OUTPUT_FILE_NAME}' file is up to date."
+              false
+            else
+              puts 'Planned Not Tested Requirements set has changed.'
+              true
+            end
+          else
+            puts "No existing #{PLANNED_NOT_TESTED_OUTPUT_FILE_NAME}."
+            true
+          end
+
+        if udpate_planned_not_tested
+          puts "Writing to file #{PLANNED_NOT_TESTED_OUTPUT_FILE}..."
+          File.write(PLANNED_NOT_TESTED_OUTPUT_FILE, new_planned_not_tested_csv, encoding: Encoding::UTF_8)
+        end
+
+        puts 'Done.'
+      end
+
+      def run_check(input_directory)
+        @input_directory = input_directory
+        check_presence_of_input_files
+
+        requirements_ok =
+          if File.exist?(REQUIREMENTS_OUTPUT_FILE)
+            if old_requirements_csv == new_requirements_csv
+              puts "'#{REQUIREMENTS_OUTPUT_FILE_NAME}' file is up to date."
+              true
+            else
+              puts "#{REQUIREMENTS_OUTPUT_FILE_NAME} file is out of date."
+              false
+            end
+          else
+            puts "No existing #{REQUIREMENTS_OUTPUT_FILE_NAME} file."
+            false
+          end
+
+        planned_not_tested_requirements_ok =
+          if File.exist?(PLANNED_NOT_TESTED_OUTPUT_FILE)
+            if old_planned_not_tested_csv == new_planned_not_tested_csv
+              puts "'#{PLANNED_NOT_TESTED_OUTPUT_FILE_NAME}' file is up to date."
+              true
+            else
+              puts "#{PLANNED_NOT_TESTED_OUTPUT_FILE_NAME} file is out of date."
+              false
+            end
+          else
+            puts "No existing #{PLANNED_NOT_TESTED_OUTPUT_FILE_NAME} file."
+            false
+          end
+
+        return if planned_not_tested_requirements_ok && requirements_ok
+
+        puts <<~MESSAGE
+          Check Failed. To resolve, run:
+
+                bundle exec rake "requirements:collect[<input_directory>]"
+
+        MESSAGE
+        exit(1)
+      end
+
+      def check_presence_of_input_files
+        input_requirement_sets.each do |req_set_id, rows|
+          next unless rows.nil?
+
+          puts %(
+            Could not find input file for set #{req_set_id} in directory #{@input_directory}. Aborting requirements
+            collection..."
+          )
+          exit(1)
+        end
+      end
+
+      def spreadsheet_value_falsy?(str)
+        %w[no false].include?(str&.downcase)
+      end
+    end
+  end
+end

data/lib/inferno_requirements_tools/tasks/requirements_coverage.rb

@@ -0,0 +1,284 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'yaml'
+require_relative '../ext/inferno_core/runnable'
+
+module InfernoRequirementsTools
+  module Tasks
+    # This class manages the mapping of test kit tests to requirements that they verify
+    # and creates a CSV file with the tests that cover each requirement.
+    # It expects a CSV file in the repo at `lib/[test kit id]/requirements/[test kit id]_requirements.csv`
+    # that serves as the source of the requirement set for the test kit. The requirements in
+    # this files are identified by a requirement set and an id and tests, groups, and suites
+    # within in the test kit can claim that they verify a requirement by including a reference
+    # to that requirementin the form <requirement set>@<id> in their `verifies_requirements` field.
+    # Requirements that are out of scope can be listed in a companion file
+    # `lib/[test kit id]/requirements/[test kit id]_out_of_scope_requirements.csv`.
+    #
+    # The `run` method generates a CSV file at
+    # `lib/[test kit id]/requirements/generated/[test kit id]_requirements_coverage.csv``.
+    # This file will be identical to the input spreadsheet, plus an additional column which holds a comma separated
+    # list of inferno test IDs that test each requirement. These test IDs are Inferno short form IDs that represent the
+    # position of the test within its group and suite. For example, the fifth test in the second group will have an ID
+    # of 2.05. This ID is also shown in the Inferno web UI.
+    #
+    # The `run_check` method will check whether the previously generated file is up-to-date.
+    class RequirementsCoverage
+      VERSION = '0.2.0' # update when making meaningful changes to this method for tracking used versions
+      CONFIG = YAML.load_file(File.join('lib', 'requirements_config.yaml'))
+
+      TEST_KIT_ID = CONFIG['test_kit_id']
+      TEST_SUITES = CONFIG['suites'].map do |test_suite|
+        Object.const_get(test_suite['class_name'])
+      end
+
+      SUITE_ID_TO_ACTOR_MAP = CONFIG['suites'].each_with_object({}) do |test_suite, hash|
+        hash[test_suite['id']] = test_suite['suite_actor']
+      end
+
+      # Derivative constants
+      TEST_KIT_CODE_FOLDER = TEST_KIT_ID
+      DASHERIZED_TEST_KIT_ID = TEST_KIT_ID.gsub('_', '-')
+      INPUT_HEADERS = [
+        'Req Set',
+        'ID',
+        'URL',
+        'Requirement',
+        'Conformance',
+        'Actor',
+        'Sub-Requirement(s)',
+        'Conditionality'
+      ].freeze
+      SHORT_ID_HEADER = 'Short ID(s)'
+      FULL_ID_HEADER = 'Full ID(s)'
+      INPUT_FILE_NAME = "#{DASHERIZED_TEST_KIT_ID}_requirements.csv".freeze
+      INPUT_FILE = File.join('lib', TEST_KIT_CODE_FOLDER, 'requirements', INPUT_FILE_NAME).freeze
+      NOT_TESTED_FILE_NAME = "#{DASHERIZED_TEST_KIT_ID}_out_of_scope_requirements.csv".freeze
+      NOT_TESTED_FILE = File.join('lib', TEST_KIT_CODE_FOLDER, 'requirements', NOT_TESTED_FILE_NAME).freeze
+      OUTPUT_HEADERS = INPUT_HEADERS + TEST_SUITES.flat_map do |suite|
+                                         ["#{suite.title} #{SHORT_ID_HEADER}", "#{suite.title} #{FULL_ID_HEADER}"]
+                                       end
+      OUTPUT_FILE_NAME = "#{DASHERIZED_TEST_KIT_ID}_requirements_coverage.csv".freeze
+      OUTPUT_FILE_DIRECTORY = File.join('lib', TEST_KIT_CODE_FOLDER, 'requirements', 'generated')
+      OUTPUT_FILE = File.join(OUTPUT_FILE_DIRECTORY, OUTPUT_FILE_NAME).freeze
+
+      def input_rows
+        @input_rows ||=
+          CSV.parse(File.open(INPUT_FILE, 'r:bom|utf-8'), headers: true).map do |row|
+            row.to_h.slice(*INPUT_HEADERS)
+          end
+      end
+
+      def not_tested_requirements_map
+        @not_tested_requirements_map ||= load_not_tested_requirements
+      end
+
+      def load_not_tested_requirements
+        return {} unless File.exist?(NOT_TESTED_FILE)
+
+        not_tested_requirements = {}
+        CSV.parse(File.open(NOT_TESTED_FILE, 'r:bom|utf-8'), headers: true).each do |row|
+          row_hash = row.to_h
+          not_tested_requirements["#{row_hash['Req Set']}@#{row_hash['ID']}"] = row_hash
+        end
+
+        not_tested_requirements
+      end
+
+      # Of the form:
+      # {
+      #     'req-id-1': [
+      #       { short_id: 'short-id-1', full_id: 'long-id-1', suite_id: 'suite-id-1' },
+      #       { short_id: 'short-id-2', full_id: 'long-id-2', suite_id: 'suite-id-2' }
+      #     ],
+      #     'req-id-2': [{ short_id: 'short-id-3', full_id: 'long-id-3', suite_id: 'suite-id-3' }],
+      #     ...
+      # }
+      def inferno_requirements_map
+        @inferno_requirements_map ||= TEST_SUITES.each_with_object({}) do |suite, requirements_map|
+          serialize_requirements(suite, 'suite', suite.id, requirements_map)
+          suite.groups.each do |group|
+            map_group_requirements(group, suite.id, requirements_map)
+          end
+        end
+      end
+
+      def new_csv
+        @new_csv ||=
+          CSV.generate(+"\xEF\xBB\xBF") do |csv|
+            csv << OUTPUT_HEADERS
+            input_rows.each do |row| # NOTE: use row order from source file
+              next if row['Conformance'] == 'DEPRECATED' # filter out deprecated rows
+
+              TEST_SUITES.each do |suite|
+                suite_actor = SUITE_ID_TO_ACTOR_MAP[suite.id]
+                if row['Actor']&.include?(suite_actor)
+                  add_suite_tests_for_row(row, suite)
+                else
+                  row["#{suite.title} #{SHORT_ID_HEADER}"] = 'NA'
+                  row["#{suite.title} #{FULL_ID_HEADER}"] = 'NA'
+                end
+              end
+              csv << row.values
+            end
+          end
+      end
+
+      def add_suite_tests_for_row(row, suite)
+        set_and_req_id = "#{row['Req Set']}@#{row['ID']}"
+        items = get_items_for_requirement(set_and_req_id, suite)
+        short_ids = items[0]
+        full_ids = items[1]
+        if short_ids.blank? && not_tested_requirements_map.key?(set_and_req_id)
+          row["#{suite.title} #{SHORT_ID_HEADER}"] = 'Not Tested'
+          row["#{suite.title} #{FULL_ID_HEADER}"] = 'Not Tested'
+        else
+          row["#{suite.title} #{SHORT_ID_HEADER}"] = short_ids&.join(', ')
+          row["#{suite.title} #{FULL_ID_HEADER}"] = full_ids&.join(', ')
+        end
+      end
+
+      def get_items_for_requirement(set_and_req_id, suite)
+        suite_requirement_items = inferno_requirements_map[set_and_req_id]&.filter do |item|
+          item[:suite_id] == suite.id
+        end
+        [
+          suite_requirement_items&.map { |item| item[:short_id] },
+          suite_requirement_items&.map { |item| item[:full_id] }
+        ]
+      end
+
+      def input_requirement_ids
+        @input_requirement_ids ||= input_rows.map { |row| "#{row['Req Set']}@#{row['ID']}" }
+      end
+
+      # The requirements present in Inferno that aren't in the input spreadsheet
+      def unmatched_requirements_map
+        @unmatched_requirements_map ||= inferno_requirements_map.except(*input_requirement_ids)
+      end
+
+      def old_csv
+        @old_csv ||= File.read(OUTPUT_FILE)
+      end
+
+      def run
+        unless File.exist?(INPUT_FILE)
+          puts "Could not find input file: #{INPUT_FILE}. Aborting requirements coverage generation..."
+          exit(1)
+        end
+
+        if unmatched_requirements_map.any?
+          puts "WARNING: The following requirements indicated in the test kit are not present in #{INPUT_FILE_NAME}"
+          output_requirements_map_table(unmatched_requirements_map)
+        end
+
+        if File.exist?(OUTPUT_FILE)
+          if old_csv == new_csv
+            puts "'#{OUTPUT_FILE_NAME}' file is up to date."
+            return
+          else
+            puts 'Requirements coverage has changed.'
+          end
+        else
+          puts "No existing #{OUTPUT_FILE_NAME}."
+        end
+
+        puts "Writing to file #{OUTPUT_FILE}..."
+        FileUtils.mkdir_p(OUTPUT_FILE_DIRECTORY)
+        File.write(OUTPUT_FILE, new_csv)
+        puts 'Done.'
+      end
+
+      def run_check
+        unless File.exist?(INPUT_FILE)
+          puts "Could not find input file: #{INPUT_FILE}. Aborting requirements coverage check..."
+          exit(1)
+        end
+
+        if unmatched_requirements_map.any?
+          puts "The following requirements indicated in the test kit are not present in #{INPUT_FILE_NAME}"
+          output_requirements_map_table(unmatched_requirements_map)
+        end
+
+        if File.exist?(OUTPUT_FILE)
+          if old_csv == new_csv
+            puts "'#{OUTPUT_FILE_NAME}' file is up to date."
+            return unless unmatched_requirements_map.any?
+          else
+            puts <<~MESSAGE
+              #{OUTPUT_FILE_NAME} file is out of date.
+              To regenerate the file, run:
+
+                  bundle exec rake requirements:generate_coverage
+
+            MESSAGE
+          end
+        else
+          puts <<~MESSAGE
+            No existing #{OUTPUT_FILE_NAME} file.
+            To generate the file, run:
+
+                  bundle exec rake requirements:generate_coverage
+
+          MESSAGE
+        end
+
+        puts 'Check failed.'
+        exit(1)
+      end
+
+      def map_group_requirements(group, suite_id, requirements_map)
+        serialize_requirements(group, group.short_id, suite_id, requirements_map)
+        group.tests&.each { |test| serialize_requirements(test, test.short_id, suite_id, requirements_map) }
+        group.groups&.each { |subgroup| map_group_requirements(subgroup, suite_id, requirements_map) }
+      end
+
+      def serialize_requirements(runnable, short_id, suite_id, requirements_map)
+        runnable.verifies_requirements&.each do |requirement_id|
+          requirement_id_string = requirement_id.to_s
+
+          requirements_map[requirement_id_string] ||= []
+          requirements_map[requirement_id_string] << { short_id:, full_id: runnable.id, suite_id: }
+        end
+      end
+
+      # Output the requirements in the map like so:
+      #
+      # requirement_id | short_id   | full_id
+      # ---------------+------------+----------
+      # req-id-1       | short-id-1 | full-id-1
+      # req-id-2       | short-id-2 | full-id-2
+      #
+      def output_requirements_map_table(requirements_map)
+        headers = %w[requirement_id short_id full_id]
+        col_widths = headers.map(&:length)
+        col_widths[0] = [col_widths[0], requirements_map.keys.map(&:length).max].max
+        col_widths[1] = ([col_widths[1]] + requirements_map.values.flatten.map { |item| item[:short_id].length }).max
+        col_widths[2] = ([col_widths[2]] + requirements_map.values.flatten.map { |item| item[:full_id].length }).max
+        col_widths.map { |width| width + 3 }
+
+        puts [
+          headers[0].ljust(col_widths[0]),
+          headers[1].ljust(col_widths[1]),
+          headers[2].ljust(col_widths[2])
+        ].join(' | ')
+        puts col_widths.map { |width| '-' * width }.join('-+-')
+        output_requirements_map_table_contents(requirements_map, col_widths)
+        puts
+      end
+
+      def output_requirements_map_table_contents(requirements_map, col_widths)
+        requirements_map.each do |requirement_id, runnables|
+          runnables.each do |runnable|
+            puts [
+              requirement_id.ljust(col_widths[0]),
+              runnable[:short_id].ljust(col_widths[1]),
+              runnable[:full_id].ljust(col_widths[2])
+            ].join(' | ')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/requirements_config.yaml

@@ -0,0 +1,17 @@
+test_kit_id: davinci_pdex_test_kit
+
+suites: 
+  - id: pdex_payer_server
+    class_name: DaVinciPDexTestKit::PDexPayerServerSuite
+    suite_actor: Server
+  - id: pdex_payer_client
+    class_name: DaVinciPDexTestKit::PDexPayerClientSuite
+    suite_actor: Client
+
+requirement_sets:
+  - id: hl7.fhir.us.davinci-pdex_2.0.0
+    folder: PDex
+    requirements: All
+    actor_map:
+      - {spec: Server, test_kit: Server}
+      - {spec: Client, test_kit: Client}