testscript_engine 0.0.0

data/lib/testscript_engine/testreport_handler.rb

@@ -0,0 +1,247 @@
+# frozen_string_literal: true
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+#                                                                               #
+#                           TestReportHandler Module                            #
+#                                                                               #
+# The TestReportHandler (handler) module is intended to be imported into the    #
+# TestScriptRunnable (runnable) class. The handler instantiates a               #
+# TestReportBuilder (builder) object tailored to the parent runnable instance.  #
+# In executing a runnable, calls (i.e. 'Pass', 'Fail') are made to the handler  #
+# module -- which then directs the builder instance to update its report        #
+# accordingly. Think of the handler as the 'API' to interact with the           #
+# TestReport output by the execution of a runnable. Each time the runnable is   #
+# executed, it instantiates a new builder instance, using the initial builder   #
+# as a template.                                                                #
+#                                                                               #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+module TestReportHandler
+  def testreport
+    report_builder.report
+  end
+
+  def fresh_testreport
+    @report_builder = nil
+  end
+
+  def report_builder
+    @report_builder ||= fresh_builder
+  end
+
+  def fresh_builder
+    builder_template.clone
+  end
+
+  def builder_template
+    @builder_template ||= TestReportBuilder.new(script)
+  end
+
+  # TODO: Remove!
+  def pass(message = nil)
+    report_builder.pass
+  end
+
+  def fail(message = nil)
+    report_builder.fail(message)
+  end
+
+  def skip(message = nil)
+    report_builder.skip(message)
+  end
+
+  def warning(message = nil)
+    report_builder.warning(message)
+  end
+
+  def error(message = nil)
+    report_builder.error(message)
+  end
+
+  def finalize_report
+    report_builder.finalize_report
+    testreport
+  end
+
+  def cascade_skips(number_to_skip)
+     while number_to_skip > 0
+      report_builder.skip
+      number_to_skip -= 1
+    end
+  end
+
+  # A 'script' method ought to be defined in the klass
+  # that includes the handler - if 'script' is undefined,
+  # this feeds an empty testscript to the builder
+  def script
+    begin
+      super
+    rescue NoMethodError
+      FHIR::TestScript.new
+    end
+  end
+
+  class TestReportBuilder
+    attr_accessor :pass_count, :total_test_count, :actions
+
+    def actions
+      @actions ||= []
+    end
+
+    def report
+      @report ||= FHIR::TestReport.new
+    end
+
+    def initialize(testscript_blueprint = nil)
+      add_boilerplate(testscript_blueprint)
+      build_setup(testscript_blueprint.setup)
+      build_test(testscript_blueprint.test)
+      build_teardown(testscript_blueprint.teardown)
+
+      self.pass_count = 0
+      self.total_test_count = actions.length
+    end
+
+    def build_setup(setup_blueprint)
+      return unless setup_blueprint
+
+      actions = setup_blueprint.action.map { |action| build_action(action) }
+      report.setup = FHIR::TestReport::Setup.new(action: actions)
+    end
+
+    def build_test(test_blueprint)
+      return if test_blueprint.empty?
+
+      report.test = test_blueprint.map do |test|
+        actions = test.action.map { |action| build_action(action) }
+        FHIR::TestReport::Test.new(action: actions)
+      end
+    end
+
+    def build_teardown(teardown_blueprint)
+      return unless teardown_blueprint
+
+      actions = teardown_blueprint.action.map { |action| build_action(action) }
+      report.teardown = FHIR::TestReport::Teardown.new(action: actions)
+    end
+
+    def build_action(action_blueprint)
+      phase = action_blueprint.class.to_s.split("::")[2]
+
+      action_definition = {
+        id: action_blueprint.id,
+        operation: build_operation(action_blueprint.operation),
+        assert: (build_assert(action_blueprint.assert) unless phase == 'Teardown')
+      }
+
+      "FHIR::TestReport::#{phase}::Action".constantize.new(action_definition)
+    end
+
+    def build_operation(operation_blueprint)
+      return unless operation_blueprint
+
+      operation_def = {
+        id: operation_blueprint.label || operation_blueprint.id || 'unlabeled operation',
+        message: operation_blueprint.description
+      }
+
+      operation = FHIR::TestReport::Setup::Action::Operation.new(operation_def)
+      store_action(operation)
+      operation
+    end
+
+    def build_assert(assert_blueprint)
+      return unless assert_blueprint
+
+      assert_def = {
+        id: assert_blueprint.label || assert_blueprint.id || 'unlabeled assert',
+        message: assert_blueprint.description
+      }
+
+      assert = FHIR::TestReport::Setup::Action::Assert.new(assert_def)
+      store_action(assert)
+      assert
+    end
+
+    def add_boilerplate(testscript_blueprint)
+      report.result = 'pending'
+      report.status = 'in-progress'
+      report.tester = 'The MITRE Corporation'
+      report.id = testscript_blueprint.id&.gsub(/(?i)testscript/, 'testreport')
+      report.name = testscript_blueprint.name&.gsub(/(?i)testscript/, 'TestReport')
+      report.testScript = FHIR::Reference.new({
+        reference: testscript_blueprint.url,
+        type: "http://hl7.org/fhir/R4B/testscript.html"
+      })
+    end
+
+    def finalize_report
+      report.issued = DateTime.now.to_s
+      report.status = 'completed'
+      report.score = (self.pass_count.to_f / self.total_test_count).round(2) * 100
+      report.result = (report.score == 100.0 ? 'pass' : 'fail')
+    end
+
+    def action
+      actions.first
+    end
+
+    def store_action(action)
+      actions.concat(Array(action))
+    end
+
+    def next_action
+      actions.shift
+      finalize_report if actions.empty?
+    end
+
+    def pass
+      action.result = 'pass'
+      self.pass_count += 1
+      next_action
+    end
+
+    def skip(message = nil)
+      action.result = 'skip'
+      action.message = message if message
+      self.total_test_count -= 1
+      next_action
+    end
+
+    def warning(message = nil)
+      action.result = 'warning'
+      action.message = message if message
+      next_action
+    end
+
+    def fail(message = nil)
+      action.result = 'fail'
+      action.message = message if message
+      next_action
+    end
+
+    def error(message = nil)
+      action.result = 'error'
+      action.message = message if message
+      next_action
+    end
+
+    def clone
+      builder_dup = self.deep_dup
+      builder_dup.actions.clear
+      builder_dup.instance_eval('@report = FHIR::TestReport.new(self.report.to_hash)')
+
+      clone_actions(builder_dup.report.setup, builder_dup)
+      builder_dup.report.test.each { |test| clone_actions(test, builder_dup) }
+      clone_actions(builder_dup.report.teardown, builder_dup)
+
+      builder_dup
+    end
+
+    def clone_actions(report_phase, clone)
+      report_phase.try(:action)&.each do |action|
+        clone.store_action(action.operation || action.assert)
+      end
+    end
+  end
+end

data/lib/testscript_engine/testscript_runnable.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+require_relative 'operation'
+require_relative 'assertion'
+require_relative 'message_handler'
+require_relative 'testreport_handler'
+
+class TestScriptRunnable
+  include Operation
+  include Assertion
+  prepend MessageHandler
+  include TestReportHandler
+
+  attr_accessor :script, :client, :reply
+
+  def id_map
+    @id_map ||= {}
+  end
+
+  def fixtures
+    @fixtures ||= {}
+  end
+
+  def request_map
+    @request_map ||= {}
+  end
+
+  def response_map
+    @response_map ||= {}
+  end
+
+  def autocreate
+    @autocreate ||= []
+  end
+
+  def autodelete_ids
+    @autodelete_ids ||= []
+  end
+
+  def initialize(script)
+    raise ArgumentError.new(messages(:bad_script)) unless script.is_a?(FHIR::TestScript)
+    raise ArgumentError.new(messages(:invalid_script_input)) unless script.valid?
+
+    @script = script
+    load_fixtures
+  end
+
+  def run(client)
+    @client = client
+
+    fresh_testreport
+
+    preprocess
+    setup
+    test
+    teardown
+    postprocessing
+
+    finalize_report
+  end
+
+  def preprocess
+    return info(:no_preprocess) if autocreate.empty?
+    autocreate.each do |fixture|
+      begin
+        client.send(*build_request((create_operation(fixture))))
+      rescue => e
+        error(:uncaught_error, e.message)
+      end
+    end
+  end
+
+  def setup
+    return info(:no_setup) unless script.setup
+    handle_actions(script.setup.action, true)
+  end
+
+  def test
+    script.test.each { |test| handle_actions(test.action, false) }
+  end
+
+  def teardown
+    return info(:no_teardown) unless script.teardown
+    handle_actions(script.teardown.action, false)
+  end
+
+  def postprocessing
+    return info(:no_postprocess) if autocreate.empty?
+
+    autodelete_ids.each do |fixture_id|
+      begin
+        client.send(*build_request((delete_operation(fixture_id))))
+      rescue => e
+        error(:uncaught_error, e.message)
+      end
+    end
+
+    @ended = nil
+    @id_map = {}
+    @request_map = {}
+    @response_map = {}
+  end
+
+  def handle_actions(actions, end_on_fail)
+    return abort_test(actions) if @ended
+    @modify_report = true
+    current_action = 0
+
+    begin
+      actions.each do |action|
+        current_action += 1
+        if action.operation
+          execute(action.operation)
+        elsif action.respond_to?(:assert)
+          begin
+            evaluate(action.assert)
+          rescue AssertionException => ae
+            if ae.outcome == :skip
+              skip(:eval_assert_result, ae.details)
+            elsif ae.outcome == :fail
+              next warning(:eval_assert_result, ae.details) if action.assert.warningOnly
+              if end_on_fail
+                @ended = true
+                fail(:eval_assert_result, ae.details)
+                cascade_skips_with_message(actions, current_action) unless current_action == actions.length
+                return
+              else
+                fail(:eval_assert_result, ae.details)
+              end
+            end
+          end
+        end
+      end
+    rescue OperationException => oe
+      error(oe.details)
+      if end_on_fail
+        @ended = true
+        cascade_skips_with_message(actions, current_action) unless current_action == actions.length
+      end
+    rescue => e
+      error(:uncaught_error, e.message)
+      cascade_skips_with_message(actions, current_action) unless current_action == actions.length
+    end
+
+    @modify_report = false
+  end
+
+  def cascade_skips_with_message(actions, current_action)
+    actions_to_skip = actions.slice(current_action, actions.length)
+    cascade_skips(:skip_on_fail, actions_to_skip, actions_to_skip.length)
+  end
+
+  def abort_test(actions_to_skip)
+    cascade_skips(:abort_test, actions_to_skip, 'setup', actions_to_skip.length)
+  end
+
+  def load_fixtures
+    script.fixture.each do |fixture|
+      next warning(:no_static_fixture_id) unless fixture.id
+      next warning(:no_static_fixture_resource) unless fixture.resource
+
+      resource = get_resource_from_ref(fixture.resource)
+      next warning(:bad_static_fixture_reference) unless resource
+
+      fixtures[fixture.id] = resource
+      autocreate << fixture.id if fixture.autocreate
+      autodelete_ids << fixture.id if fixture.autodelete
+    end
+  end
+
+  def get_resource_from_ref(reference)
+    return warning(:bad_reference) unless reference.is_a?(FHIR::Reference)
+
+    ref = reference.reference
+    return warning(:no_reference) unless ref
+    return warning(:unsupported_ref, ref) if ref.start_with? 'http'
+
+    if ref.start_with? '#'
+      contained = script.contained.find { |r| r.id == ref[1..] }
+      return contained || warning(:no_contained_resource, ref)
+    end
+
+    begin
+      fixture_path = script.url.split('/')[0...-1].join('/') + '/fixtures'
+      filepath = File.expand_path(ref, File.absolute_path(fixture_path))
+      file = File.open(filepath, 'r:UTF-8', &:read)
+      file.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+      resource = FHIR.from_contents(file)
+      info(:loaded_static_fixture, ref, script.id)
+      return resource
+    rescue => e
+      warning(:resource_extraction, ref, e.message)
+    end
+  end
+
+  def storage(op)
+    @reply = client.reply
+    reply.nil? ? return : client.reply = nil
+
+    request_map[op.requestId] = reply.request if op.requestId
+    response_map[op.responseId] = reply.response if op.responseId
+
+    (reply.resource = FHIR.from_contents(reply.response&.[](:body).to_s)) rescue {}
+    (reply.response[:body] = reply.resource)
+    response_map[op.responseId][:body] = reply.resource if reply.resource and response_map[op.responseId]
+
+    if op.targetId and (reply.request[:method] == :delete) and [200, 201, 204].include?(reply.response[:code])
+      id_map.delete(op.targetId) and return
+    end
+
+    dynamic_id = reply.resource&.id || begin
+      reply.response&.[](:headers)&.[]('location')&.remove(reply.request[:url].to_s)&.split('/')&.[](2)
+    end
+
+    id_map[op.responseId] = dynamic_id if op.responseId and dynamic_id
+    id_map[op.sourceId] = dynamic_id if op.sourceId and dynamic_id
+    return
+  end
+
+  def find_resource id
+    fixtures[id] || response_map[id]&.[](:body)
+  end
+
+  def replace_variables placeholder
+    return placeholder unless placeholder&.include? '${'
+    replaced = placeholder.clone
+
+    script.variable.each do |var|
+      next unless replaced.include? "${#{var.name}}"
+      replacement = evaluate_variable(var)
+      replaced.gsub!("${#{var.name}}", replacement) if replacement
+    end
+
+    return replaced
+  end
+
+  def evaluate_variable var
+    if var.expression
+      evaluate_expression(var.expression, find_resource(var.sourceId))
+    elsif var.path
+      evaluate_path(var.path, find_resource(var.sourceId))
+    elsif var.headerField
+      headers = response_map[var.sourceId]&.[](:headers)
+      headers&.find { |h, v| h == var.headerField || h == var.headerField.downcase }&.last
+    end || var.defaultValue
+  end
+
+  def evaluate_expression(expression, resource)
+    return unless expression and resource
+
+    return FHIRPath.evaluate(expression, resource.to_hash)
+  end
+end

data/lib/testscript_engine.rb

@@ -0,0 +1,134 @@
+require 'pry-nav'
+require 'fhir_client'
+require 'fhir_models'
+require_relative 'testscript_engine/testscript_runnable'
+require_relative 'testscript_engine/message_handler'
+
+class TestScriptEngine
+  prepend MessageHandler
+
+  attr_accessor :endpoint, :testscript_path, :testreport_path
+
+  def scripts
+    @scripts ||= {}
+  end
+
+  def runnables
+    @runnables ||= {}
+  end
+
+  def reports
+    @reports ||= {}
+  end
+
+  def client
+    @client ||= begin
+      info(:begin_initialize_client)
+      client = FHIR::Client.new(endpoint || 'localhost:3000')
+      info(:finish_initialize_client)
+      client
+    end
+  end
+
+  def initialize(endpoint, testscript_path, testreport_path)
+    self.endpoint = endpoint
+    self.testscript_path = testscript_path
+    self.testreport_path = testreport_path
+    self.debug_mode = true
+  end
+
+  # TODO: Tie-in stronger validation. Possibly, Inferno validator.
+  def valid_testscript? script
+    return (script.is_a? FHIR::TestScript) && script.valid?
+  end
+
+  # @path [String] Optional, specifies the path to the folder containing the
+  #                TestScript Resources to-be loaded into the engine.
+  def load_scripts
+		if File.file?(testscript_path)
+			on_deck = [testscript_path]
+		elsif File.directory?(testscript_path)
+			on_deck = Dir.glob (["#{testscript_path}/**/*.{json}", "#{testscript_path}/**/*.{xml}"])
+		end
+    on_deck.each do |resource|
+      next if resource.include? "/fixtures/"
+
+      begin
+        script = FHIR.from_contents File.read(resource)
+        if valid_testscript? script
+          script.url = resource
+          if scripts[script.id]
+            info(:overwrite_existing_script, script.id)
+          else
+            info(:loaded_script, script.id)
+          end
+          scripts[script.id] = script
+        else
+          info(:invalid_script, resource)
+        end
+      rescue
+        info(:bad_serialized_script, resource)
+      end
+    end
+  end
+
+  # @script [FHIR::TestScript] Optional, a singular TestScript resource to be
+  #                            transformed into a runnable. If no resource is
+  #                            given, all stored TestScript are by default
+  #                            transformed into and stored as runnables.
+  def make_runnables script = nil
+    begin
+      if valid_testscript? script
+        runnables[script.id] = TestScriptRunnable.new script
+        info(:created_runnable, script.id)
+      else
+        scripts.each do |_, script|
+          runnables[script.id] = TestScriptRunnable.new script
+          info(:created_runnable, script.id)
+        end
+      end
+    rescue => e
+      error(:unable_to_create_runnable, script.id)
+    end
+  end
+
+  # TODO: Clean-up, possibly modularize into a pretty_print type method
+  # @runnable_id [String] Optional, specifies the id of the runnable to be
+  #                       tested against the endpoint.
+  def execute_runnables runnable_id = nil
+    if runnable_id
+      if runnables[runnable_id]
+        reports[runnable_id] = runnables[runnable_id].run(client)
+      else
+        error(:unable_to_locate_runnable, runnable_id)
+      end
+    else
+      runnables.each do |id, runnable|
+        reports[id] = runnable.run(client)
+      end
+    end
+  end
+
+	def verify_runnable(runnable_id)
+		return true unless runnables[runnable_id].nil?
+		false
+	end
+
+	def new_client(url)
+		@client = nil
+		@endpoint = url
+	end
+
+  # @path [String] Optional, specifies the path to the folder which the
+  #                TestReport resources should be written to.
+  def write_reports path = nil
+    report_directory = path || testreport_path
+    FileUtils.mkdir_p report_directory
+
+    reports.each do |_, report|
+      File.open("#{report_directory}/#{report.name.downcase.split(' ')[1...].join('_')}.json", 'w') do |f|
+        f.write(report.to_json)
+      end
+    end
+  end
+end