bulk_data_test_kit 0.13.2 → 0.14.0

data/lib/bulk_data_test_kit/v4.0.0/submit/provider/docs/suite_description.md

@@ -0,0 +1,50 @@
+The Bulk Data Access v4.0.0-ballot Submit - Provider test suite validates a Data Provider
+implementation of the [Bulk Submit Operation](https://hl7.org/fhir/uv/bulkdata/4.0.0-202609-ballot/en/)
+within the Bulk Data Access v4.0.0-ballot IG.
+
+## Scope
+
+These tests target the
+[Bulk Data Access v4.0.0-ballot IG](https://hl7.org/fhir/uv/bulkdata/4.0.0-202609-ballot/en/).
+The ballot specification is subject to change before final publication.
+These tests are intended to allow implementers to perform
+preliminary checks of their systems against the requirements stated for Bulk Submit actors
+and [provide feedback](https://github.com/inferno-framework/bulk-data-test-kit/issues)
+on the tests. Future versions of these tests may verify other
+requirements and may change the test verification logic.
+
+## Running the Tests
+
+### Quick Start
+
+Bulk data access requires the use of SMART Backend Services for authentication.
+In order to interact with Inferno's simulated bulk server, the tester must provide
+the JSON Web Key Set (JWKS) containing the asymmetric signing key in the
+**SMART JSON Web Key Set (JWKS)** input as either a URL that resolves
+to a JWKS or a raw JWKS in JSON format. Additionally, testers may provide
+a **Client Id** if they want their client assigned a specific one.
+
+Once the client has been registered, it will need to obtain an access token
+following the SMART Backend Services flow and use it when making bulk data requests.
+If the client is not able to obtain an access token, see the *Demonstration* section
+below for how to use the SMART server tests to obtain an access token that the client can use.
+
+### Demonstration
+
+You can run the Data Provider and Data Consumer suites against each other to
+see how the tests work:
+
+1. Start a session in the Bulk Data Access v4.0.0-ballot Submit - Provider suite.
+1. Select the "Demo: Run Data Provider Against Inferno Data Consumer" preset.
+1. Click the "Run All Tests" button.
+1. In another browser tab, start a session in the Bulk Data Access v4.0.0-ballot
+   Submit - Consumer suite.
+1. Select the "Demo: Run Data Consumer Against Inferno Data Provider" preset.
+1. Click the "Run All Tests" button.
+1. When the Data Consumer suite pauses after sending `completed`, advance the
+   Data Provider suite from its initial wait.
+1. The Data Provider suite will retrieve the manifest and file synchronously,
+   then pause while waiting for the final status poll.
+1. Advance the Data Consumer suite. It will make one final status poll.
+1. After that poll completes, advance the Data Provider suite to finish the
+   paired workflow.

data/lib/bulk_data_test_kit/v4.0.0/submit/provider/endpoints/poll.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'smart_app_launch_test_kit'
+require_relative '../helpers'
+
+module BulkDataTestKit
+  module BulkDataV400
+    module Submit
+      module Provider
+        module Endpoints
+          class Poll < Inferno::DSL::SuiteEndpoint
+            include URLs
+            include Helpers
+
+            def test_run_identifier
+              return request.params[:session_path] if request.params[:session_path].present?
+
+              SMARTAppLaunch::MockSMARTServer.issued_token_to_client_id(
+                request.headers['authorization']&.delete_prefix('Bearer ')
+              )
+            end
+
+            def make_response
+              case submission_outcome[:status]
+              when 'succeeded'
+                make_completed_response
+              when 'failed'
+                make_error_response(500, submission_outcome[:diagnostic])
+              else
+                make_in_progress_response
+              end
+            end
+
+            def submission_id
+              submission_outcome[:submission_id]
+            end
+
+            def submission_outcome
+              @submission_outcome ||= submission_outcomes
+                .fetch(request.params[:submission_key].to_s, {})
+                .deep_symbolize_keys
+            end
+
+            def submission_outcomes
+              raw_outcomes = Inferno::Repositories::SessionData.new.load(
+                test_session_id: test_run.test_session_id,
+                name: :provider_submission_outcomes
+              )
+
+              JSON.parse(raw_outcomes.presence || '{}')
+            rescue JSON::ParserError
+              {}
+            end
+
+            def make_in_progress_response
+              response.status = 202
+              response.headers['Retry-After'] = '1'
+            end
+
+            def make_completed_response
+              response.status = 200
+              response.headers['Content-Type'] = 'application/json'
+              response.body = complete_manifest(submission_id).to_json
+            end
+
+            def make_error_response(status, diagnostic)
+              operation_outcome = FHIR::R4::OperationOutcome.new(
+                issue: [
+                  FHIR::R4::OperationOutcome::Issue.new(
+                    severity: 'error',
+                    code: 'processing',
+                    diagnostics: diagnostic
+                  )
+                ]
+              )
+
+              response.status = status
+              response.headers['Content-Type'] = 'application/fhir+json'
+              response.body = operation_outcome.to_json
+            end
+
+            def tags
+              [POLL_TAG]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bulk_data_test_kit/v4.0.0/submit/provider/endpoints/status.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'smart_app_launch_test_kit'
+require_relative '../helpers'
+require_relative '../../shared/submission_status'
+
+module BulkDataTestKit
+  module BulkDataV400
+    module Submit
+      module Provider
+        module Endpoints
+          class Status < Inferno::DSL::SuiteEndpoint
+            include URLs
+            include Helpers
+
+            def test_run_identifier
+              return request.params[:session_path] if request.params[:session_path].present?
+
+              SMARTAppLaunch::MockSMARTServer.issued_token_to_client_id(
+                request.headers['authorization']&.delete_prefix('Bearer ')
+              )
+            end
+
+            def make_response
+              if identity.nil?
+                response.status = 400
+                response.headers['Content-Type'] = 'application/fhir+json'
+                response.body = FHIR::R4::OperationOutcome.new(
+                  issue: [
+                    FHIR::R4::OperationOutcome::Issue.new(
+                      severity: 'error',
+                      code: 'invalid',
+                      diagnostics: 'The request did not identify a Bulk Submit submission.'
+                    )
+                  ]
+                ).to_json
+              else
+                response.status = 202
+                response.headers['Content-Location'] = poll_url(submission_key(identity))
+              end
+            end
+
+            def identity
+              @identity ||= submission_identity(request_parameters)
+            end
+
+            def request_parameters
+              body = request.body.read
+              request.body.rewind
+              FHIR.from_contents(body)
+            rescue JSON::ParserError, TypeError
+              nil
+            end
+
+            def tags
+              [STATUS_SUBMIT_TAG]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bulk_data_test_kit/v4.0.0/submit/provider/endpoints/submit.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative '../tags'
+
+require 'smart_app_launch_test_kit'
+
+module BulkDataTestKit
+  module BulkDataV400
+    module Submit
+      module Provider
+        module Endpoints
+          class Submit < Inferno::DSL::SuiteEndpoint
+            include URLs
+
+            def test_run_identifier
+              return request.params[:session_path] if request.params[:session_path].present?
+
+              SMARTAppLaunch::MockSMARTServer.issued_token_to_client_id(
+                request.headers['authorization']&.delete_prefix('Bearer ')
+              )
+            end
+
+            def make_response
+              response.status = 200
+            end
+
+            def tags
+              [SUBMIT_TAG]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bulk_data_test_kit/v4.0.0/submit/provider/helpers.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+require 'digest'
+require_relative '../shared/submission_status'
+
+module BulkDataTestKit
+  module BulkDataV400
+    module Submit
+      module Provider
+        module Helpers
+          TERMINAL_SUBMISSION_STATUSES = %w[completed stopped].freeze
+
+          def submission_identity(parameters)
+            parameter_list = parameters.respond_to?(:parameter) ? parameters.parameter : []
+            submitter = parameter_list.find { |parameter| parameter.name == 'submitter' }&.valueIdentifier
+            submission_id = parameter_list.find do |parameter|
+              parameter.name == 'submissionId'
+            end&.valueString
+
+            return if submitter&.system.blank? || submitter&.value.blank? || submission_id.blank?
+
+            {
+              submitter_system: submitter.system,
+              submitter_value: submitter.value,
+              submission_id:
+            }
+          end
+
+          def submission_status(parameters)
+            parameter_list = parameters.respond_to?(:parameter) ? parameters.parameter : []
+            status_parameters = parameter_list.select do |parameter|
+              parameter.name == 'submissionStatus'
+            end
+            return unless status_parameters.one?
+
+            coding = status_parameters.first.valueCoding
+
+            return unless coding&.system == Shared::SubmissionStatus::SYSTEM
+
+            coding.code
+          end
+
+          def parsed_submission_parameters(requests)
+            requests.filter_map do |request|
+              parameters = FHIR.from_contents(request.request_body)
+              parameters if parameters.is_a?(FHIR::R4::Parameters)
+            rescue JSON::ParserError, TypeError
+              nil
+            end
+          end
+
+          def submission_timelines(parameters_list)
+            parameters_list.each_with_object({}) do |parameters, timelines|
+              identity = submission_identity(parameters)
+              next unless identity
+
+              (timelines[identity] ||= []) << parameters
+            end
+          end
+
+          def valid_completed_submission_timelines(parameters_list)
+            submission_timelines(parameters_list).select do |_identity, timeline|
+              terminal_indexes = timeline.each_index.select do |index|
+                TERMINAL_SUBMISSION_STATUSES.include?(submission_status(timeline[index]))
+              end
+
+              terminal_indexes == [timeline.length - 1] &&
+                submission_status(timeline.last) == 'completed'
+            end
+          end
+
+          def valid_completed_submission_parameters(requests)
+            valid_completed_submission_timelines(
+              parsed_submission_parameters(requests)
+            ).values.flatten
+          end
+
+          def validate_terminal_submission_order(requests)
+            assert requests.present?, 'No $bulk-submit requests were received.'
+
+            submission_timelines(parsed_submission_parameters(requests)).each_value do |timeline|
+              terminal_index = timeline.index do |parameters|
+                TERMINAL_SUBMISSION_STATUSES.include?(submission_status(parameters))
+              end
+
+              assert terminal_index.nil? || terminal_index == timeline.length - 1,
+                     'No `$bulk-submit` request may follow a terminal submission status ' \
+                     'for the same `submitter` and `submissionId`.'
+            end
+          end
+
+          def submission_key(identity)
+            Digest::SHA256.hexdigest(
+              [identity[:submitter_system], identity[:submitter_value], identity[:submission_id]].to_json
+            )
+          end
+
+          def complete_manifest(submission_id)
+            {
+              submissionId: submission_id,
+              transactionTime: DateTime.now,
+              requiresAccessToken: true
+            }
+          end
+
+          def validate_submitter_identifiers(requests, operation_name)
+            assert requests.present?, "No #{operation_name} requests were received."
+
+            requests.each do |request|
+              parameters = FHIR.from_contents(request.request_body)
+              submitter_parameters = parameters.parameter.select do |parameter|
+                parameter.name == 'submitter'
+              end
+
+              assert submitter_parameters.one?,
+                     "Every #{operation_name} request must include exactly one `submitter` parameter."
+
+              submitter_identifier = submitter_parameters.first.valueIdentifier
+              assert submitter_identifier.present?,
+                     'The `submitter` parameter must have type `Identifier`.'
+              assert submitter_identifier.system.present?,
+                     'The `submitter` Identifier must include a system.'
+              assert submitter_identifier.value.present?,
+                     'The `submitter` Identifier must include a value.'
+
+              [submitter_identifier.system, submitter_identifier.value]
+            end
+          end
+
+          def validate_status_submission_identities(status_requests, submit_requests)
+            validate_submitter_identifiers(status_requests, '$bulk-submit-status')
+            validate_submission_ids(status_requests, '$bulk-submit-status')
+            validate_submitter_identifiers(submit_requests, '$bulk-submit')
+            validate_submission_ids(submit_requests, '$bulk-submit')
+
+            submitted_identities = submit_requests.map do |request|
+              submission_identity(FHIR.from_contents(request.request_body))
+            end
+            status_identities = status_requests.map do |request|
+              submission_identity(FHIR.from_contents(request.request_body))
+            end
+
+            assert status_identities.all? { |identity| submitted_identities.include?(identity) },
+                   'Every `$bulk-submit-status` request must match the `submitter` and ' \
+                   '`submissionId` of a `$bulk-submit` request.'
+          end
+
+          def validate_submission_ids(requests, operation_name)
+            assert requests.present?, "No #{operation_name} requests were received."
+
+            requests.each do |request|
+              parameters = FHIR.from_contents(request.request_body)
+              submission_id_parameters = parameters.parameter.select do |parameter|
+                parameter.name == 'submissionId'
+              end
+
+              assert submission_id_parameters.one?,
+                     "Every #{operation_name} request must include exactly one `submissionId` parameter."
+              assert submission_id_parameters.first.valueString.present?,
+                     'The `submissionId` parameter must have type `string` and contain a value.'
+            end
+          end
+
+          def submission_status_parameters(request)
+            FHIR.from_contents(request.request_body).parameter.select do |parameter|
+              parameter.name == 'submissionStatus'
+            end
+          end
+
+          def request_has_submission_status?(request, code)
+            status_parameters = submission_status_parameters(request)
+            return code == 'in-progress' if status_parameters.empty?
+            return false unless status_parameters.one?
+
+            coding = status_parameters.first.valueCoding
+            coding.present? &&
+              coding.system == Shared::SubmissionStatus::SYSTEM &&
+              coding.code == code
+          end
+
+          def validate_submission_statuses(requests)
+            assert requests.present?, 'No $bulk-submit requests were received.'
+
+            requests.each do |request|
+              status_parameters = submission_status_parameters(request)
+
+              assert status_parameters.length <= 1,
+                     'Every $bulk-submit request must include at most one `submissionStatus` parameter.'
+              next if status_parameters.empty?
+
+              coding = status_parameters.first.valueCoding
+              assert coding.present?, '`submissionStatus` must have type `Coding`.'
+              assert Shared::SubmissionStatus::CODES.include?(coding.code),
+                     "#{coding.code} is not a valid submissionStatus code"
+              assert coding.system == Shared::SubmissionStatus::SYSTEM,
+                     "#{coding.system} is not the submissionStatus code system"
+            end
+          end
+
+          def validate_optional_part_parameter(requests, parameter_name)
+            parameters = requests.flat_map do |request|
+              FHIR.from_contents(request.request_body).parameter
+            end.select do |parameter|
+              parameter.name == parameter_name
+            end
+
+            skip_if parameters.empty?,
+                    "No submission included the optional `#{parameter_name}` parameter"
+
+            all_parts = parameters.all? do |parameter|
+              serialized = parameter.to_hash
+              parameter.part.present? &&
+                serialized.keys.none? { |key| key.start_with?('value') || key == 'resource' }
+            end
+
+            assert all_parts, "`#{parameter_name}` must have type `part`"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bulk_data_test_kit/v4.0.0/submit/provider/manifest_and_file_retrieval.rb

@@ -0,0 +1,165 @@
+require_relative 'tags'
+
+module BulkDataTestKit
+  module BulkDataV400
+    module Submit
+      module Provider
+        class ManifestAndFileRetrieval < Inferno::TestGroup
+          include Helpers
+
+          title 'Manifest and File Retrieval'
+          id :bulk_data_v400_submit_provider_manifest_file_retrieval
+
+          input :provider_submission_outcomes, type: :textarea, optional: true
+
+          test do
+            id :bulk_data_v400_submit_provider_completed_submissions_processed
+            title 'Completed submissions were processed successfully'
+            description <<~DESCRIPTION
+              This test verifies that Inferno successfully retrieved every
+              completed submission.
+            DESCRIPTION
+
+            run do
+              outcomes = JSON.parse(provider_submission_outcomes.presence || '{}')
+              skip_if outcomes.empty?, 'No completed submissions were processed'
+
+              failures = outcomes.values.select { |outcome| outcome['status'] == 'failed' }
+              assert failures.empty?, 'Inferno was unable to retrieve one or more completed submissions.'
+            end
+          end
+
+          test do
+            id :bulk_data_v400_submit_provider_oauth_metadata_retrieved
+            title 'OAuth metadata retrieved'
+            description <<~DESCRIPTION
+              This test verifies that the provider's OAuth metadata could be
+              retrieved.
+            DESCRIPTION
+
+            run do
+              submit_requests = load_tagged_requests Provider::SUBMIT_TAG
+              completed_submissions = valid_completed_submission_parameters(submit_requests)
+              oauth_metadata_submitted = completed_submissions.any? do |parameters|
+                manifest_submitted = parameters.parameter.any? do |parameter|
+                  parameter.name == 'manifestUrl' && parameter.valueUrl.present?
+                end
+                oauth_metadata_url_submitted = parameters.parameter.any? do |parameter|
+                  parameter.name == 'oauthMetadataUrl' && parameter.valueUrl.present?
+                end
+
+                manifest_submitted && oauth_metadata_url_submitted
+              end
+
+              skip_if !oauth_metadata_submitted, 'No OAuth metadata URL was submitted'
+
+              oauth_metadata_requests = load_tagged_requests Provider::OAUTH_METADATA_TAG
+
+              assert oauth_metadata_requests.present?, 'No OAuth Metadata request made'
+
+              failed_requests = oauth_metadata_requests.reject { |request| request.status.to_i == 200 }
+              assert failed_requests.empty?,
+                     "OAuth metadata retrieval failed: #{failed_requests.map { |request| "#{request.url} returned #{request.status}" }.join(', ')}"
+            end
+          end
+
+          test do
+            id :bulk_data_v400_submit_provider_token_request_success
+            title 'Successful token request made'
+            description <<~DESCRIPTION
+              This test verifies that all files included in the manifest
+              were retrieved.
+            DESCRIPTION
+
+            run do
+              submit_requests = load_tagged_requests Provider::SUBMIT_TAG
+              completed_submissions = valid_completed_submission_parameters(submit_requests)
+              oauth_metadata_submitted = completed_submissions.any? do |parameters|
+                manifest_submitted = parameters.parameter.any? do |parameter|
+                  parameter.name == 'manifestUrl' && parameter.valueUrl.present?
+                end
+                oauth_metadata_url_submitted = parameters.parameter.any? do |parameter|
+                  parameter.name == 'oauthMetadataUrl' && parameter.valueUrl.present?
+                end
+
+                manifest_submitted && oauth_metadata_url_submitted
+              end
+
+              skip_if !oauth_metadata_submitted, 'No OAuth metadata URL was submitted'
+
+              token_requests = load_tagged_requests Provider::TOKEN_TAG
+
+              assert token_requests.present?, 'No token request made'
+
+              failed_requests = token_requests.reject { |request| request.status.to_i == 200 }
+              assert failed_requests.empty?,
+                     "Token request failed: #{failed_requests.map { |request| "#{request.url} returned #{request.status}" }.join(', ')}"
+            end
+          end
+
+          test do
+            id :bulk_data_v400_submit_provider_manifests_retrieved
+            title 'Manifests for each submission were retrieved'
+            description <<~DESCRIPTION
+              This test verifies that all manifests included in the submissions
+              were retrieved.
+            DESCRIPTION
+
+            run do
+              submit_requests = load_tagged_requests Provider::SUBMIT_TAG
+              completed_submissions = valid_completed_submission_parameters(submit_requests)
+              expected_manifest_count = completed_submissions.sum do |parameters|
+                parameters.parameter.count do |parameter|
+                  parameter.name == 'manifestUrl' && parameter.valueUrl.present?
+                end
+              end
+
+              skip_if expected_manifest_count.zero?, 'No manifest URLs were submitted'
+
+              manifest_requests = load_tagged_requests Provider::MANIFEST_TAG
+
+              assert manifest_requests.count == expected_manifest_count,
+                     "Expected #{expected_manifest_count} manifest retrievals, " \
+                     "but received #{manifest_requests.count}"
+
+              failed_requests = manifest_requests.reject { |request| request.status.to_i == 200 }
+              assert failed_requests.empty?,
+                     "Manifest retrieval failed: #{failed_requests.map { |request| "#{request.url} returned #{request.status}" }.join(', ')}"
+            end
+          end
+
+          test do
+            id :bulk_data_v400_submit_provider_files_retrieved
+            title 'Files retrieved'
+            description <<~DESCRIPTION
+              This test verifies that all files included in the manifest
+              were retrieved.
+            DESCRIPTION
+
+            run do
+              manifest_requests = load_tagged_requests Provider::MANIFEST_TAG
+
+              expected_file_count = manifest_requests
+                .select { |request| request.status.to_i == 200 }
+                .sum do |request|
+                  manifest = JSON.parse(request.response_body)
+                  Array(manifest['output']).count
+                end
+
+              skip_if expected_file_count.zero?, 'No files were listed in the retrieved manifests'
+
+              file_requests = load_tagged_requests Provider::FILE_DOWNLOAD_TAG
+
+              assert file_requests.count == expected_file_count,
+                     "Expected #{expected_file_count} file downloads, but received #{file_requests.count}"
+
+              failed_requests = file_requests.reject { |request| request.status.to_i == 200 }
+              assert failed_requests.empty?,
+                     "File retrieval failed: #{failed_requests.map { |request| "#{request.url} returned #{request.status}" }.join(', ')}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bulk_data_test_kit/v4.0.0/submit/provider/poll_group.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module BulkDataTestKit
+  module BulkDataV400
+    module Submit
+      module Provider
+        class PollGroup < Inferno::TestGroup
+          title 'Bulk Data Status Polling Request'
+
+          description %(
+            This group verifies that a provider successfully polled for status.
+          )
+
+          id :bulk_data_v400_submit_provider_poll
+
+          run_as_group
+
+          optional
+
+          test do
+            id :bulk_data_v400_submit_provider_poll_request
+            title 'Poll Request Was Made'
+
+            description %(
+              This test verifies that at least one poll request was made.
+            )
+
+            run do
+              assert load_tagged_requests(POLL_TAG).any?, 'Did not receive a poll request.'
+            end
+          end
+        end
+      end
+    end
+  end
+end