plan_executor 1.0.2 → 1.8.0

data/lib/tasks/tasks.rake

@@ -25,11 +25,13 @@ namespace :crucible do
 
   desc 'console'
   task :console, [] do |t, args|
+    FHIR.logger = Logger.new("logs/plan_executor.log", 10, 1024000)
     binding.pry
   end
 
   desc 'execute all'
   task :execute_all, [:url, :html_summary] do |t, args|
+    FHIR.logger = Logger.new("logs/plan_executor.log", 10, 1024000)
     require 'benchmark'
     b = Benchmark.measure {
       client = FHIR::Client.new(args.url)
@@ -45,6 +47,7 @@ namespace :crucible do
 
   desc 'execute all test scripts'
   task :execute_all_testscripts, [:url, :html_summary] do |t, args|
+    FHIR.logger = Logger.new("logs/plan_executor.log", 10, 1024000)
     require 'benchmark'
     b = Benchmark.measure {
       client = FHIR::Client.new(args.url)
@@ -59,8 +62,34 @@ namespace :crucible do
     puts "Execute All completed in #{b.real} seconds."
   end
 
+  desc 'execute testscript and get testreport'
+  task :testreport, [:url, :test, :filename] do |t, args|
+    FHIR.logger = Logger.new("logs/plan_executor.log", 10, 1024000)
+    require 'benchmark'
+    b = Benchmark.measure {
+      client = FHIR::Client.new(args.url)
+      options = client.get_oauth2_metadata_from_conformance
+      set_client_secrets(client,options) unless options.empty?
+      engine = Crucible::Tests::TestScriptEngine.new(client)
+      script = engine.find_test(args.test)
+      if script.nil?
+        puts "Unable to find TestScript #{args.test}"
+      else
+        results = script.execute
+        if args.filename
+          f = File.open(args.filename,'w:UTF-8')
+          f.write(results.values.first.to_json)
+          f.close
+        end
+        puts results.values.first.to_json
+      end
+    }
+    puts "TestReport completed in #{b.real} seconds."
+  end
+
   desc 'execute'
   task :execute, [:url, :test, :resource] do |t, args|
+    FHIR.logger = Logger.new("logs/plan_executor.log", 10, 1024000)
     require 'benchmark'
     b = Benchmark.measure {
       client = FHIR::Client.new(args.url)
@@ -72,9 +101,10 @@ namespace :crucible do
   end
 
   desc 'metadata'
-  task :metadata, [:url, :test] do |t, args|
+  task :metadata, [:test] do |t, args|
+    FHIR.logger = Logger.new("logs/plan_executor.log", 10, 1024000)
     require 'benchmark'
-    b = Benchmark.measure { collect_metadata(FHIR::Client.new(args.url), args.test) }
+    b = Benchmark.measure { puts JSON.pretty_unparse(Crucible::Tests::Executor.new(nil).extract_metadata_from_test(args.test)) }
     puts "Metadata #{args.test} completed in #{b.real} seconds."
   end
 
@@ -97,23 +127,28 @@ namespace :crucible do
   def execute_test(client, key, resourceType=nil)
     executor = Crucible::Tests::Executor.new(client)
     test = executor.find_test(key)
-    if test.nil?
+    if test.nil? || (test.is_a?(Array) && test.empty?)
       puts "Unable to find test: #{key}"
       return
     end
     results = nil
-    if !resourceType.nil? && test.respond_to?(:resource_class=)
-      fhir_classes = Mongoid.models.select {|c| c.name.include? 'FHIR'}
-      klass = fhir_classes.find{|x|x.to_s.include?(resourceType)}
-      results = test.execute(klass) if !klass.nil?
+    if !resourceType.nil? && test.respond_to?(:resource_class=) && FHIR::RESOURCES.include?(resourceType)
+      results = test.execute("FHIR::#{resourceType}".constantize)
     end
     results = executor.execute(test) if results.nil?
     output_results results
   end
 
   def execute_all(client)
-    results = Crucible::Tests::Executor.new(client).execute_all
-    output_results results
+    executor = Crucible::Tests::Executor.new(client)
+    all_results = {}
+    executor.tests.each do |test|
+      next if test.multiserver
+      results = executor.execute(test)
+      all_results.merge! results
+      output_results results
+    end
+    all_results
   end
 
   def execute_multiserver_test(client, client2, key)
@@ -121,17 +156,15 @@ namespace :crucible do
     output_results executor.execute(executor.find_test(key))
   end
 
-  def collect_metadata(client, test)
-    output_results Crucible::Tests::Executor.new(client).metadata(test), true
-  end
-
   def output_results(results, metadata_only=false)
-    require 'turn'
+    require 'ansi'
     results.keys.each do |suite_key|
       puts suite_key
-      results[suite_key].each do |test|
+      suite = results[suite_key]
+      suite = convert_testreport_to_testresults(suite) if suite.is_a?(FHIR::TestReport)
+      suite.each do |test|
         puts write_result(test['status'], test[:test_method], test['message'])
-        if test['status'].upcase=='ERROR'
+        if test['status'].upcase=='ERROR' && test['data']
           puts " "*12 + "-"*40
           puts " "*12 + "#{test['data'].gsub("\n","\n"+" "*12)}"
           puts " "*12 + "-"*40
@@ -151,6 +184,61 @@ namespace :crucible do
     results
   end
 
+  def convert_testreport_to_testresults(testreport)
+    results = []
+
+    if testreport.setup
+      statuses = Hash.new(0)
+      message = nil
+      testreport.setup.action.each do |action|
+        if action.operation
+          statuses[action.operation.result] += 1
+          message = action.operation.message if ['fail','error','skip'].include?(action.operation.result) && message.nil? && action.operation.message
+        elsif action.assert
+          statuses[action.assert.result] += 1
+          message = action.assert.message if ['fail','error','skip'].include?(action.assert.result) && message.nil? && action.assert.message
+        end
+      end
+      if statuses['error'] > 0
+        status = 'error'
+      elsif statuses['fail'] > 0
+        status = 'fail'
+      elsif statuses['skip'] > 0
+        status = 'skip'
+      else
+        status = 'pass'
+      end
+      results << Crucible::Tests::TestResult.new('SETUP', 'Setup for TestScript', status, message, nil).to_hash
+      results.last[:test_method] = 'SETUP'
+    end
+
+    testreport.test.each do |test|
+      statuses = Hash.new(0)
+      message = nil
+      test.action.each do |action|
+        if action.operation
+          statuses[action.operation.result] += 1
+          message = action.operation.message if ['fail','error','skip'].include?(action.operation.result) && message.nil? && action.operation.message
+        elsif action.assert
+          statuses[action.assert.result] += 1
+          message = action.assert.message if ['fail','error','skip'].include?(action.assert.result) && message.nil? && action.assert.message
+        end
+      end
+      if statuses['error'] > 0
+        status = 'error'
+      elsif statuses['fail'] > 0
+        status = 'fail'
+      elsif statuses['skip'] > 0
+        status = 'skip'
+      else
+        status = 'pass'
+      end
+      results << Crucible::Tests::TestResult.new(test.name, test.description, status, message, nil).to_hash
+      results.last[:test_method] = test.name
+    end
+    results
+  end
+
   def generate_html_summary(url, results, id="summary")
     require 'erb'
     require 'tilt'
@@ -187,6 +275,7 @@ namespace :crucible do
 
   desc 'execute custom'
   task :execute_custom, [:test, :resource_type, :html_summary] do |t, args|
+    FHIR.logger = Logger.new("logs/plan_executor.log", 10, 1024000)
     require 'benchmark'
 
     puts "# #{args.test}"
@@ -215,6 +304,7 @@ namespace :crucible do
 
   desc 'execute all custom'
   task :execute_all_custom, [:html_summary] do |t, args|
+    FHIR.logger = Logger.new("logs/plan_executor.log", 10, 1024000)
     require 'benchmark'
 
     puts "# #{args.test}"
@@ -264,7 +354,7 @@ namespace :crucible do
   end
 
   desc 'list all test scripts'
-  task :list_all_test_scripts do
+  task :list_testscripts do
     require 'benchmark'
     b = Benchmark.measure { puts Crucible::Tests::TestScriptEngine.list_all.keys }
     puts "List all tests completed in #{b.real} seconds."
@@ -272,7 +362,8 @@ namespace :crucible do
 
   desc 'execute with requirements'
   task :execute_w_requirements, [:url, :test] do |t, args|
-    require 'turn'
+    FHIR.logger = Logger.new("logs/plan_executor.log", 10, 1024000)
+    require 'ansi'
 
     module Crucible
       module Tests
@@ -305,13 +396,53 @@ namespace :crucible do
 
   def write_result(status, test_name, message)
     tab_size = 10
-    "#{' '*(tab_size - status.length)}#{Turn::Colorize.method(status.to_sym).call(status.upcase)} #{test_name}: #{message}"
+    "#{' '*(tab_size - status.length)}#{colorize(status)} #{test_name}: #{message}"
+  end
+
+  def colorize(status)
+    case status.upcase
+    when 'PASS'
+      ANSI.green{ status.upcase }
+    when 'SKIP'
+      ANSI.blue{ status.upcase }
+    when 'FAIL'
+      ANSI.red{ status.upcase }
+    else
+      ANSI.white_on_red{ status.upcase }
+    end
+  end
+
+  desc 'update fixtures from spec'
+  task :update_fixtures, [:publish_folder] do |t, args|
+
+    # open publish folder
+    # open fixtures
+    # go through each folder within fixtures
+    # if file exists in publish folder, replace it
+
+    root = File.expand_path '../..', File.dirname(File.absolute_path(__FILE__))
+    fixtures = File.join(root, 'fixtures')
+    publish = File.join(args.publish_folder)
+
+    files = Dir.glob(File.join(fixtures, '**', '*.xml'))
+    files.each do |file|
+      basename = File.basename(file)
+      updated_file = File.join(publish, basename)
+      if File.exists?(updated_file)
+        puts "Updating Fixture: #{basename}..."
+        FileUtils.copy updated_file, file 
+      else
+        puts "Unable to update fixture: #{basename}"
+      end
+    end
+
   end
 
   namespace :multiserver do
     desc 'execute'
     task :execute, [:url1, :url2, :test] do |t, args|
-      require 'turn'
+      FHIR.logger = Logger.new("logs/multiserver.log", 10, 1024000)
+      require 'ansi'
       require 'benchmark'
       b = Benchmark.measure {
         client1 = FHIR::Client.new(args.url1)

data/lib/tests/assertions.rb

@@ -19,52 +19,52 @@ module Crucible
         case operator
         when :equals
           unless assertion_negated( expected == actual )
-            message += " Expected: #{expected}, but found: #{actual}."
+            message += " Expected #{expected} but found #{actual}."
             raise AssertionException.new message, data
           end
         when :notEquals
           unless assertion_negated( expected != actual )
-            message += " Expected not: #{expected}, but found: #{actual}."
+            message += " Did not expect #{expected} but found #{actual}."
             raise AssertionException.new message, data
           end
         when :in
           unless assertion_negated(expected.split(",").include?(actual))
-            message += " Expected #{actual} to be in #{expected}."
+            message += " Expected #{expected} but found #{actual}."
             raise AssertionException.new message, data
           end
         when :notIn
           unless assertion_negated(!expected.split(",").include?(actual))
-            message += " Expected #{actual} to not be in: #{expected}."
+            message += " Did not expect #{expected} but found #{actual}."
             raise AssertionException.new message, data
           end
         when :greaterThan
           unless assertion_negated(!actual.nil? && !expected.nil? && actual > expected)
-            message += " Expected #{actual} to be greater than #{expected}."
+            message += " Expected greater than #{expected} but found #{actual}."
             raise AssertionException.new message, data
           end
         when :lessThan
           unless assertion_negated(!actual.nil? && !expected.nil? && actual < expected)
-            message += " Expected #{actual} to be greater than #{expected}."
+            message += " Expected less than #{expected} but found #{actual}."
             raise AssertionException.new message, data
           end
         when :empty
           unless assertion_negated(actual.nil? || actual.length == 0)
-            message += " Expected #{actual} to be empty."
+            message += " Expected empty but found #{actual}."
             raise AssertionException.new message, data
           end
         when :notEmpty
           unless assertion_negated(!actual.nil? && actual.length > 0)
-            message += " Expected #{actual} to not be empty."
+            message += " Expected not empty but found #{actual}."
             raise AssertionException.new message, data
           end
         when :contains
-          unless assertion_negated(actual.include?(expected))
-            message += " Expected #{actual} to contain #{actual}."
+          unless assertion_negated(actual && actual.include?(expected))
+            message += " Expected #{actual} to contain #{expected}."
             raise AssertionException.new message, data
           end
         when :notContains
-          unless assertion_negated(!actual.include?(expected))
-            message += " Expected #{actual} to not contain #{actual}."
+          unless assertion_negated(actual.nil? || !actual.include?(expected))
+            message += " Expected #{actual} to not contain #{expected}."
             raise AssertionException.new message, data
           end
         else
@@ -76,7 +76,7 @@ module Crucible
 
       def assert_valid_profile(response, klass)
         unless assertion_negated( response[:code].to_s == "200")
-          
+
           raise AssertionException.new "Server created a #{klass.name.demodulize} with the ID `_validate` rather than validate the resource." if response[:code].to_s == "201"
 
           raise AssertionException.new "Response code #{response[:code]} with no OperationOutcome provided"
@@ -114,6 +114,12 @@ module Crucible
         end
       end
 
+      def assert_response_conflict(response)
+        unless assertion_negated( [409, 412].include?(response.code) )
+          raise AssertionException.new "Bad response code: expected 409 or 412, but found #{response.code}", response.body
+        end
+      end
+
       def assert_navigation_links(bundle)
         unless assertion_negated( bundle.first_link && bundle.last_link && bundle.next_link )
           raise AssertionException.new "Expecting first, next and last link to be present"
@@ -125,7 +131,7 @@ module Crucible
           # check what this is...
           found = response.resource
           begin
-            found = FHIR::Resource.from_contents(response.body)
+            found = FHIR.from_contents(response.body)
           rescue
             found = nil
           end
@@ -142,7 +148,7 @@ module Crucible
       def assert_bundle_transactions_okay(response)
         response.resource.entry.each do |entry|
           unless assertion_negated( !entry.response.nil? )
-            raise AssertionException.new "All Transaction/Batch Bundle.entry elements SHALL have a response." 
+            raise AssertionException.new "All Transaction/Batch Bundle.entry elements SHALL have a response."
           end
           status = entry.response.status
           unless assertion_negated( status && status.start_with?('200','201','204') )
@@ -156,10 +162,9 @@ module Crucible
         response_content_type = header
         response_content_type = header[0, header.index(';')] if !header.index(';').nil?
 
-        unless assertion_negated( "application/#{content_type}+fhir" == response_content_type )
-          raise AssertionException.new "Expected a #{content_type} content-type", response_content_type
+        unless assertion_negated( "application/fhir+#{content_type}" == response_content_type )
+          raise AssertionException.new "Expected content-type application/fhir+#{content_type} but found #{response_content_type}", response_content_type
         end
-
       end
 
       # Based on MIME Types defined in
@@ -250,6 +255,7 @@ module Crucible
       attr_accessor :data
       def initialize(message, data=nil)
         super(message)
+        FHIR.logger.error "AssertionException: #{message}"
         @data = data
       end
     end

data/lib/tests/base_test.rb

@@ -3,13 +3,15 @@ module Crucible
     class BaseTest
 
       include Crucible::Tests::Assertions
+      require 'method_source'
 
-      BASE_SPEC_LINK = 'http://hl7.org/fhir/DSTU2'
+      BASE_SPEC_LINK = 'http://hl7.org/fhir/2017Jan'
       REST_SPEC_LINK = "#{BASE_SPEC_LINK}/http.html"
 
       attr_accessor :tests_subset
       attr_accessor :tags
       attr_accessor :category
+      attr_accessor :warnings
 
       # Base test fields, used in Crucible::Tests::Executor.list_all
       JSON_FIELDS = ['author','description','id','tests','title', 'multiserver', 'tags', 'details', 'category']
@@ -23,10 +25,12 @@ module Crucible
 
       def initialize(client, client2=nil)
         @client = client
+        FHIR::Resource.new.client = client
         @client2 = client2
         @client.monitor_requests if @client
         @client2.monitor_requests if @client2
         @tags ||= []
+        @warnings = []
       end
 
       def multiserver
@@ -55,14 +59,17 @@ module Crucible
         methods = tests & @tests_subset unless @tests_subset.blank?
         methods.each do |test_method|
           @client.requests = [] if @client
-          puts "[#{title}#{('_' + @resource_class.name.demodulize) if @resource_class}] #{prefix}: #{test_method}..."
+          FHIR.logger.info "[#{title}#{('_' + @resource_class.name.demodulize) if @resource_class}] #{prefix}: #{test_method}..."
           begin
             result << execute_test_method(test_method)
           rescue => e
             result << TestResult.new('ERROR', "Error #{prefix} #{test_method}", STATUS[:error], "#{test_method} failed, fatal error: #{e.message}", e.backtrace.join("\n")).to_hash.merge!({:test_method => test_method})
           end
         end
-        teardown if respond_to? :teardown and not @metadata_only
+        begin
+          teardown if respond_to? :teardown and not @metadata_only
+        rescue
+        end
         result
       end
 
@@ -112,6 +119,13 @@ module Crucible
         end
       end
 
+      def ignore_client_exception
+        begin
+          yield
+        rescue ClientException
+        end
+      end
+
     end
   end
 end