bulk_data_test_kit 0.9.0

data/lib/bulk_data_test_kit/v1.0.1/bulk_data_test_suite.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require_relative '../version'
+require_relative 'bulk_data_smart_backend_services_group'
+require_relative 'bulk_data_export_tests_test_group'
+
+module BulkDataTestKit
+  module BulkDataV101
+    class BulkDataTestSuite < Inferno::TestSuite
+      title 'Bulk Data Access v1.0.1'
+      version VERSION
+      id :bulk_data_v101
+      links [
+        {
+          label: 'Report Issue',
+          url: 'https://github.com/inferno-framework/bulk-data-test-kit/issues/'
+        },
+        {
+          label: 'Open Source',
+          url: 'https://github.com/inferno-framework/bulk-data-test-kit/'
+        },
+        {
+          label: 'Download',
+          url: 'https://github.com/inferno-framework/bulk-data-test-kit/releases'
+        }
+      ]
+
+      VALIDATION_MESSAGE_FILTERS = [
+        /Observation\.effective\.ofType\(Period\): .*vs-1:/ # Invalid invariant in FHIR v4.0.1
+      ].freeze
+
+      VERSION_SPECIFIC_MESSAGE_FILTERS = [].freeze
+
+      validator do
+        url ENV.fetch('BULK_DATA_VALIDATOR_URL', 'http://validator_service:4567')
+
+        message_filters = VALIDATION_MESSAGE_FILTERS + VERSION_SPECIFIC_MESSAGE_FILTERS
+
+        $num_messages = 0
+        $capped_message = false
+        $num_errors = 0
+        $capped_errors = false
+
+        exclude_message do |message|
+          if message.type != 'error'
+            $num_messages += 1
+          else
+            $num_errors += 1
+          end
+
+          message_filters.any? { |filter| filter.match? message.message } ||
+            (message.type != 'error' && $num_messages > 50 && !message.message.include?('Inferno is only showing the first')) ||
+            (message.type == 'error' && $num_errors > 20 && !message.message.include?('Inferno is only showing the first'))
+        end
+
+        perform_additional_validation do
+          if $num_messages > 50 && !$capped_message
+            $capped_message = true
+            { type: 'info', message: 'Inferno is only showing the first 50 validation info and warning messages.' }
+          elsif $num_errors > 20 && !$capped_errors
+            $capped_errors = true
+            { type: 'error', message: 'Inferno is only showing the first 20 validation error messages.' }
+          end
+        end
+      end
+
+      def self.jwks_json
+        bulk_data_jwks = JSON.parse(File.read(
+                                      ENV.fetch('BULK_DATA_JWKS',
+                                                File.join(File.expand_path('..', __dir__), 'bulk_data_jwks.json'))
+                                    ))
+        @jwks_json ||= JSON.pretty_generate(
+          { keys: bulk_data_jwks['keys'].select { |key| key['key_ops']&.include?('verify') } }
+        )
+      end
+
+      def self.well_known_route_handler
+        ->(_env) { [200, { 'Content-Type' => 'application/json' }, [jwks_json]] }
+      end
+
+      route(
+        :get,
+        '/.well-known/jwks.json',
+        well_known_route_handler
+      )
+
+      config(
+        inputs: {
+          client_auth_encryption_method: {
+            title: 'Client Authentication Encryption Method',
+            locked: true
+          }
+        },
+        options: {
+          post_authorization_uri: "#{Inferno::Application['base_url']}/custom/smart_stu2/post_auth"
+        }
+      )
+
+      description %(
+        The Bulk Data Access Test Kit is a testing tool that will demonstrate
+        the ability to exporting bulk data from a FHIR server to a
+        pre-authorized client following the criterion from the [FHIR Bulk Data
+        Access IG](http://hl7.org/fhir/uv/bulkdata/STU1.0.1).
+
+        This test kit is split into three different types of bulk data export:
+
+        - All Patients: FHIR Operation to obtain a detailed set of FHIR
+          resources of diverse resource types pertaining to all patients
+        - Group of Patients: FHIR Operation to obtain a detailed set of FHIR
+          resources of diverse resource types pertaining to all members of a
+          specified Group 
+        - System Level Export: FHIR Operation to export data from a FHIR
+          server, whether or not it is associated with a patient
+
+        This test kit uses [Backend Services
+        Authorization](http://hl7.org/fhir/uv/bulkdata/STU1.0.1/authorization/index.html)
+        to obtain an access token from the server. After authorization, a group
+        level, patient level, and system level bulk data export request is
+        initialized, and for each type of export, the tests read exported NDJSON
+        files from the server and validate the resources in each file. It is
+        expected the server will provide Encounter, Location, Organization, and
+        Practitioner resources as they are referenced as must support elements
+        in required resources.
+
+        To get started, if your server supports SMART backend services
+        authorization, please first register Inferno with the following JWK Set
+        URL:
+
+        * `#{Inferno::Application[:base_url]}/custom/bulk_data_v101/.well-known/jwks.json`
+
+        Then, run the full Bulk Data Access test suite containing both the SMART
+        Backend Services test group and the Bulk Data Export Tests test group.
+        Otherwise, if your server does not support SMART Backend Services
+        authorization, only run the second test group, Bulk Data Export Tests.
+      )
+
+      input_instructions %(
+        Register Inferno as a bulk data client with the following information,
+        and enter the client id and client registration in the appropriate
+        fields. For the Group export tests, please enter the group export
+        information in the appropriate box.
+
+        Register Inferno with the following JWK Set Url:
+
+        * `#{Inferno::Application[:base_url]}/custom/bulk_data_v101/.well-known/jwks.json`
+      )
+
+      input :bulk_server_url,
+            title: 'Bulk Data FHIR URL',
+            description: 'The URL of the Bulk FHIR server.'
+
+      fhir_client :bulk_server do
+        url :bulk_server_url
+      end
+
+      http_client :bulk_server do
+        url :bulk_server_url
+      end
+
+      group from: :bulk_data_smart_backend_services,
+            id: :bulk_data_smart_backend_services_v101
+
+      group from: :bulk_data_export_tests_v101
+    end
+  end
+end

data/lib/bulk_data_test_kit/v1.0.1/bulk_data_valid_resources_test.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../bulk_export_validation_tester'
+
+module BulkDataTestKit
+  module BulkDataV101
+    class BulkDataValidResourcesTest < Inferno::Test
+      include BulkDataTestKit::BulkExportValidationTester
+
+      id :bulk_data_valid_resources
+
+      title 'All resources returned are valid FHIR resources'
+      description <<~DESCRIPTION
+        This test verifies that the resources returned from bulk data export
+        conform to the base FHIR standard. This includes checking for missing
+        data elements and value set verification. This test caps the number of
+        validation info and warning messages that it will display to 50 and the
+        number of error messages it will display to 20.
+      DESCRIPTION
+
+      input :status_output
+      input :requires_access_token
+
+      run do
+        perform_bulk_export_validation(bulk_status_output: status_output,
+                                       bulk_requires_access_token: requires_access_token)
+      end
+    end
+  end
+end

data/lib/bulk_data_test_kit/v1.0.1/group/bulk_data_group_export_cancel_group.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative '../../export_kick_off_performer'
+require_relative '../bulk_data_export_cancel_test'
+
+module BulkDataTestKit
+  module BulkDataV101
+    class BulkDataGroupExportCancelGroup < Inferno::TestGroup
+      id :bulk_data_group_export_cancel_group
+      title 'Group Compartment Export Cancel Tests'
+      description %(
+        Verify that the Bulk Data server supports cancelling requested group exports.
+        This group initiates a new export and immediately cancels it to verify
+        correct behavior.
+      )
+
+      input :bearer_token
+      input :bulk_server_url,
+            title: 'Bulk Data FHIR URL',
+            description: 'The URL of the Bulk FHIR server.'
+      input :group_id,
+            title: 'Group ID',
+            description: 'The Group ID associated with the group of patients to be exported.'
+
+      test from: :bulk_data_export_cancel,
+           id: :bulk_data_group_export_cancel,
+           config: {
+             options: { resource_type: 'Group', bulk_export_url: 'Group/[group_id]/$export' }
+           }
+    end
+  end
+end

data/lib/bulk_data_test_kit/v1.0.1/group/bulk_data_group_export_group.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'tls_test_kit'
+require_relative '../bulk_data_export_operation_support_test'
+require_relative '../bulk_data_no_auth_test'
+require_relative '../bulk_data_export_kick_off_test'
+require_relative '../bulk_data_status_check_test'
+require_relative '../bulk_data_output_check_test'
+
+module BulkDataTestKit
+  module BulkDataV101
+    class BulkDataGroupExportGroup < Inferno::TestGroup
+      title 'Group Compartment Export Tests'
+      short_description 'Verify that the system supports Group compartment export.'
+      description <<~DESCRIPTION
+        Verify that group level export on the Bulk Data server follow the Bulk Data Access Implementation Guide
+      DESCRIPTION
+      id :bulk_data_group_export_group
+
+      input :bearer_token,
+            title: 'Bulk Data Authorization Bearer Token',
+            description: 'The authorization bearer token for the Bulk FHIR server. If not required, leave blank.',
+            optional: true
+      input :group_id,
+            title: 'Group ID',
+            description: 'The Group ID associated with the group of patients to be exported.'
+      input :bulk_server_url,
+            title: 'Bulk Data FHIR URL',
+            description: 'The URL of the Bulk FHIR server.'
+      input :bulk_timeout,
+            title: 'Export Times Out after (1-600)',
+            description: <<~DESCRIPTION,
+              While testing, Inferno waits for the server to complete the exporting task. If the calculated totalTime is
+              greater than the timeout value specified here, Inferno bulk client stops testing. Please enter an integer
+              for the maximum wait time in seconds. If timeout is less than 1, Inferno uses default value 180. If the
+                timeout is greater than 600 (10 minutes), Inferno uses the maximum value 600.
+            DESCRIPTION
+            default: 180
+
+      output :requires_access_token, :status_output, :bulk_download_url
+
+      run_as_group
+
+      test from: :bulk_data_export_operation_support do
+        title 'Bulk Data Server declares support for Group export operation in CapabilityStatement'
+        description <<~DESCRIPTION
+          This test verifies that the Bulk Data Server declares support for
+          `Group/[group_id]/$export` operation in its server CapabilityStatement.
+
+          Given flexibility in the FHIR specification for declaring constrained
+          OperationDefinitions, this test only verifies that the server declares
+          any operation on the Group resource.  It does not verify that it
+          declares the standard Group export OperationDefinition provided in the
+          Bulk Data specification, nor does it attempt to resolve any non-standard
+          OperationDefinitions to verify if it is a constrained version of the
+          standard OperationDefintion.
+
+          This test will provide a warning if no operations are declared at
+          `Group/[group_id]/$export`, via the
+          `CapabilityStatement.rest.resource.operation.name` element.  It will
+          also provide an informational message if an operation on the Group
+          resource exists, but does not point to the standard OperationDefinition
+          canonical URL:
+          http://hl7.org/fhir/uv/bulkdata/OperationDefinition/group-export
+
+          Additionally, this test provides a warning if the bulk data server does
+          not include the following URL in its `CapabilityStatement.instantiates`
+          element: http://hl7.org/fhir/uv/bulkdata/CapabilityStatement/bulk-data
+        DESCRIPTION
+        id :bulk_data_group_export_operation_support
+
+        config(
+          options: { resource_type: 'Group' }
+        )
+      end
+
+      test from: :bulk_data_no_auth_reject,
+           id: :bulk_data_group_no_auth_reject,
+           config: {
+             options: { resource_type: 'Group', bulk_export_url: 'Group/[group_id]/$export' }
+           }
+
+      test from: :bulk_data_kick_off,
+           id: :bulk_data_group_kick_off,
+           config: {
+             options: { resource_type: 'Group', bulk_export_url: 'Group/[group_id]/$export' }
+           }
+
+      test from: :bulk_data_status_check,
+           id: :bulk_data_group_status_check,
+           config: {
+             options: { resource_type: 'Group' }
+           }
+
+      test from: :bulk_data_output_check,
+           id: :bulk_data_group_output_check,
+           config: {
+             options: { resource_type: 'Group' }
+           }
+    end
+  end
+end

data/lib/bulk_data_test_kit/v1.0.1/group/bulk_data_group_export_validation_group.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative '../bulk_data_multiple_patients_test'
+require_relative '../bulk_data_ndjson_download_test'
+require_relative '../bulk_data_valid_resources_test'
+
+module BulkDataTestKit
+  module BulkDataV101
+    class BulkDataGroupExportValidation < Inferno::TestGroup
+      title 'Group Compartment Export Validation Tests'
+      short_description 'Verify that the data from Group export conforms to the base FHIR standard.'
+      description <<~DESCRIPTION
+        Verify that Group compartment export from the Bulk Data server follow the base FHIR standard
+      DESCRIPTION
+
+      id :bulk_data_group_export_validation
+
+      input :status_output, :requires_access_token, :bearer_token, :bulk_download_url
+      input :lines_to_validate,
+            title: 'Limit validation to a maximum resource count',
+            description: 'To validate all, leave blank.',
+            optional: true
+
+      test from: :tls_version_test do
+        title 'Bulk Data Server is secured by transport layer security'
+        description <<~DESCRIPTION
+          [§170.315(g)(10) Test Procedure]
+          (https://www.healthit.gov/test-method/standardized-api-patient-and-population-services)
+          requires that all exchanges described herein between a client and a
+          server SHALL be secured using Transport Layer Security  (TLS)
+          Protocol Version 1.2 (RFC5246).
+        DESCRIPTION
+        id :bulk_file_server_tls_version
+
+        config(
+          inputs: { url: { name: :bulk_download_url } },
+          options: { minimum_allowed_version: OpenSSL::SSL::TLS1_2_VERSION }
+        )
+      end
+
+      test from: :bulk_data_ndjson_download,
+           id: :bulk_data_group_ndjson_download
+
+      test from: :bulk_data_valid_resources,
+           id: :bulk_data_group_valid_resources
+
+      test from: :bulk_data_multiple_patients,
+           id: :bulk_data_group_multiple_patients
+    end
+  end
+end

data/lib/bulk_data_test_kit/v1.0.1/patient/bulk_data_patient_export_cancel_group.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative '../../export_kick_off_performer'
+require_relative '../bulk_data_export_cancel_test'
+
+module BulkDataTestKit
+  module BulkDataV101
+    class BulkDataPatientExportCancelGroup < Inferno::TestGroup
+      id :bulk_data_patient_export_cancel_group
+      title 'Patient Export Cancel Tests'
+      description %(
+        Verify that the Bulk Data server supports cancelling requested exports of all Patients.
+        This group initiates a new export and immediately cancels it to verify
+        correct behavior.
+      )
+
+      input :bearer_token
+      input :bulk_server_url,
+            title: 'Bulk Data FHIR URL',
+            description: 'The URL of the Bulk FHIR server.'
+
+      output :patient_cancelled_polling_url
+
+      test from: :bulk_data_export_cancel,
+           id: :bulk_data_patient_export_cancel,
+           config: {
+             outputs: { cancelled_polling_url: { name: :patient_cancelled_polling_url } },
+             options: { resource_type: 'Patient', bulk_export_url: 'Patient/$export' }
+           }
+    end
+  end
+end

data/lib/bulk_data_test_kit/v1.0.1/patient/bulk_data_patient_export_group.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'tls_test_kit'
+require_relative '../bulk_data_export_operation_support_test'
+require_relative '../bulk_data_no_auth_test'
+require_relative '../bulk_data_export_kick_off_test'
+require_relative '../bulk_data_status_check_test'
+require_relative '../bulk_data_output_check_test'
+
+module BulkDataTestKit
+  module BulkDataV101
+    class BulkDataPatientExportGroup < Inferno::TestGroup
+      title 'Patient Export Tests'
+      short_description 'Verify that the system supports bulk export of all Patients'
+      description <<~DESCRIPTION
+        Verify that patient level export on the Bulk Data server follow the Bulk Data Access Implementation Guide
+      DESCRIPTION
+      id :bulk_data_patient_export_group
+
+      input :bearer_token,
+            title: 'Bulk Data Authorization Bearer Token',
+            description: 'The authorization bearer token for the Bulk FHIR server. If not required, leave blank.',
+            optional: true
+      input :bulk_server_url,
+            title: 'Bulk Data FHIR URL',
+            description: 'The URL of the Bulk FHIR server.'
+      input :bulk_timeout,
+            title: 'Export Times Out after (1-600)',
+            description: <<~DESCRIPTION,
+              While testing, Inferno waits for the server to complete the exporting task. If the calculated totalTime is
+              greater than the timeout value specified here, Inferno bulk client stops testing. Please enter an integer
+              for the maximum wait time in seconds. If timeout is less than 1, Inferno uses default value 180. If the
+                timeout is greater than 600 (10 minutes), Inferno uses the maximum value 600.
+            DESCRIPTION
+            default: 180
+
+      output :patient_requires_access_token, :patient_status_output, :patient_bulk_download_url
+
+      test from: :bulk_data_export_operation_support do
+        title 'Bulk Data Server declares support for Patient export operation in CapabilityStatement'
+        description <<~DESCRIPTION
+          This test verifies that the Bulk Data Server declares support for
+          `Patient/$export` operation in its server CapabilityStatement.
+
+          Given flexibility in the FHIR specification for declaring constrained
+          OperationDefinitions, this test only verifies that the server declares
+          any operation on the Patient resource.  It does not verify that it
+          declares the standard Patient export OperationDefinition provided in the
+          Bulk Data specification, nor does it attempt to resolve any non-standard
+          OperationDefinitions to verify if it is a constrained version of the
+          standard OperationDefintion.
+
+          This test will provide a warning if no operations are declared at
+          `Patient/$export`, via the
+          `CapabilityStatement.rest.resource.operation.name` element.  It will
+          also provide an informational message if an operation on the Patient
+          resource exists, but does not point to the standard OperationDefinition
+          canonical URL:
+          http://hl7.org/fhir/uv/bulkdata/OperationDefinition/patient-export
+
+          Additionally, this test provides a warning if the bulk data server does
+          not include the following URL in its `CapabilityStatement.instantiates`
+          element: http://hl7.org/fhir/uv/bulkdata/CapabilityStatement/bulk-data
+        DESCRIPTION
+        id :bulk_data_patient_export_operation_support
+
+        config(
+          options: { resource_type: 'Patient' }
+        )
+      end
+
+      test from: :bulk_data_no_auth_reject,
+           id: :bulk_data_patient_no_auth_reject,
+           config: {
+             options: { resource_type: 'Patient', bulk_export_url: 'Patient/$export' }
+           }
+
+      test from: :bulk_data_kick_off,
+           id: :bulk_data_patient_kick_off,
+           config: {
+             outputs: { polling_url: { name: :patient_polling_url } },
+             options: { resource_type: 'Patient', bulk_export_url: 'Patient/$export' }
+           }
+
+      test from: :bulk_data_status_check,
+           id: :bulk_data_patient_status_check,
+           config: {
+             inputs: { polling_url: { name: :patient_polling_url } },
+             outputs: {
+               status_response: { name: :patient_status_response },
+               requires_access_token: { name: :patient_requires_access_token }
+             },
+             options: { resource_type: 'Patient' }
+           }
+
+      test from: :bulk_data_output_check,
+           id: :bulk_data_patient_output_check,
+           config: {
+             inputs: { status_response: { name: :patient_status_response } },
+             outputs: {
+               status_output: { name: :patient_status_output },
+               bulk_download_url: { name: :patient_bulk_download_url }
+             },
+             options: { resource_type: 'Patient' }
+           }
+    end
+  end
+end

data/lib/bulk_data_test_kit/v1.0.1/patient/bulk_data_patient_export_validation_group.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../bulk_data_multiple_patients_test'
+require_relative '../bulk_data_ndjson_download_test'
+require_relative '../bulk_data_valid_resources_test'
+
+module BulkDataTestKit
+  module BulkDataV101
+    class BulkDataPatientExportValidation < Inferno::TestGroup
+      title 'Patient Export Validation Tests'
+      short_description 'Verify that the data from the export of all Patients conforms to the base FHIR standard.'
+      description <<~DESCRIPTION
+        Verify that All Patient export from the Bulk Data server follow the base FHIR standard
+      DESCRIPTION
+
+      id :bulk_data_patient_export_validation
+
+      input :patient_status_output, :patient_requires_access_token, :bearer_token, :patient_bulk_download_url
+      input :lines_to_validate,
+            title: 'Limit validation to a maximum resource count',
+            description: 'To validate all, leave blank.',
+            optional: true
+
+      test from: :tls_version_test do
+        title 'Bulk Data Server is secured by transport layer security'
+        description <<~DESCRIPTION
+          [§170.315(g)(10) Test Procedure]
+          (https://www.healthit.gov/test-method/standardized-api-patient-and-population-services)
+          requires that all exchanges described herein between a client and a
+          server SHALL be secured using Transport Layer Security  (TLS)
+          Protocol Version 1.2 (RFC5246).
+        DESCRIPTION
+        id :bulk_file_server_tls_version
+
+        config(
+          inputs: { url: { name: :patient_bulk_download_url } },
+          options: { minimum_allowed_version: OpenSSL::SSL::TLS1_2_VERSION }
+        )
+      end
+
+      test from: :bulk_data_ndjson_download,
+           id: :bulk_data_patient_ndjson_download,
+           config: {
+             inputs: {
+               bulk_download_url: { name: :patient_bulk_download_url },
+               requires_access_token: { name: :patient_requires_access_token }
+             }
+           }
+
+      test from: :bulk_data_valid_resources,
+           id: :bulk_data_patient_valid_resources,
+           config: {
+             inputs: {
+               status_output: { name: :patient_status_output },
+               requires_access_token: { name: :patient_requires_access_token }
+             }
+           }
+
+      test from: :bulk_data_multiple_patients,
+           id: :bulk_data_patient_multiple_patients
+    end
+  end
+end

data/lib/bulk_data_test_kit/v1.0.1/system_export/bulk_data_system_export_cancel_group.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative '../../export_kick_off_performer'
+require_relative '../bulk_data_export_cancel_test'
+
+module BulkDataTestKit
+  module BulkDataV101
+    class BulkDataSystemExportCancelGroup < Inferno::TestGroup
+      id :bulk_data_system_export_cancel_group
+      title 'System Level Export Cancel Tests'
+      description %(
+        Verify that the Bulk Data server supports cancelling requested system level exports.
+        This group initiates a new export and immediately cancels it to verify
+        correct behavior.
+      )
+
+      input :bearer_token
+      input :bulk_server_url,
+            title: 'Bulk Data FHIR URL',
+            description: 'The URL of the Bulk FHIR server.'
+
+      output :system_cancelled_polling_url
+
+      test from: :bulk_data_export_cancel,
+           id: :bulk_data_system_export_cancel,
+           config: {
+             outputs: { cancelled_polling_url: { name: :system_cancelled_polling_url } },
+             options: { resource_type: 'system', bulk_export_url: '$export' }
+           }
+    end
+  end
+end