plan_executor 1.0.2

data/lib/tests/assertions.rb

@@ -0,0 +1,261 @@
+module Crucible
+  module Tests
+    module Assertions
+
+      def assert(test, message="assertion failed, no message", data="")
+        unless test
+          raise AssertionException.new message, data
+        end
+      end
+
+      def assert_equal(expected, actual, message="", data="")
+        unless assertion_negated( expected == actual )
+          message += " Expected: #{expected}, but found: #{actual}."
+          raise AssertionException.new message, data
+        end
+      end
+
+      def assert_operator(operator, expected, actual, message="", data="")
+        case operator
+        when :equals
+          unless assertion_negated( expected == actual )
+            message += " Expected: #{expected}, but found: #{actual}."
+            raise AssertionException.new message, data
+          end
+        when :notEquals
+          unless assertion_negated( expected != actual )
+            message += " Expected not: #{expected}, but found: #{actual}."
+            raise AssertionException.new message, data
+          end
+        when :in
+          unless assertion_negated(expected.split(",").include?(actual))
+            message += " Expected #{actual} to be in #{expected}."
+            raise AssertionException.new message, data
+          end
+        when :notIn
+          unless assertion_negated(!expected.split(",").include?(actual))
+            message += " Expected #{actual} to not be in: #{expected}."
+            raise AssertionException.new message, data
+          end
+        when :greaterThan
+          unless assertion_negated(!actual.nil? && !expected.nil? && actual > expected)
+            message += " Expected #{actual} to be greater than #{expected}."
+            raise AssertionException.new message, data
+          end
+        when :lessThan
+          unless assertion_negated(!actual.nil? && !expected.nil? && actual < expected)
+            message += " Expected #{actual} to be greater than #{expected}."
+            raise AssertionException.new message, data
+          end
+        when :empty
+          unless assertion_negated(actual.nil? || actual.length == 0)
+            message += " Expected #{actual} to be empty."
+            raise AssertionException.new message, data
+          end
+        when :notEmpty
+          unless assertion_negated(!actual.nil? && actual.length > 0)
+            message += " Expected #{actual} to not be empty."
+            raise AssertionException.new message, data
+          end
+        when :contains
+          unless assertion_negated(actual.include?(expected))
+            message += " Expected #{actual} to contain #{actual}."
+            raise AssertionException.new message, data
+          end
+        when :notContains
+          unless assertion_negated(!actual.include?(expected))
+            message += " Expected #{actual} to not contain #{actual}."
+            raise AssertionException.new message, data
+          end
+        else
+          message += " Invalid test; unknown operator: #{operator}."
+          raise AssertionExection.new message, data
+        end
+
+      end
+
+      def assert_valid_profile(response, klass)
+        unless assertion_negated( response[:code].to_s == "200")
+          
+          raise AssertionException.new "Server created a #{klass.name.demodulize} with the ID `_validate` rather than validate the resource." if response[:code].to_s == "201"
+
+          raise AssertionException.new "Response code #{response[:code]} with no OperationOutcome provided"
+        end
+
+      end
+
+      def assert_response_ok(response, error_message="")
+        unless assertion_negated( [200, 201].include?(response.code) )
+          raise AssertionException.new "Bad response code: expected 200, 201, but found #{response.code}.#{" " + error_message}", response.body
+        end
+      end
+
+      def assert_response_created(response, error_message="")
+        unless assertion_negated( [201].include?(response.code) )
+          raise AssertionException.new "Bad response code: expected 201, but found #{response.code}.#{" " + error_message}", response.body
+        end
+      end
+
+      def assert_response_gone(response)
+        unless assertion_negated( [410].include?(response.code) )
+          raise AssertionException.new "Bad response code: expected 410, but found #{response.code}", response.body
+        end
+      end
+
+      def assert_response_not_found(response)
+        unless assertion_negated( [404].include?(response.code) )
+          raise AssertionException.new "Bad response code: expected 404, but found #{response.code}", response.body
+        end
+      end
+
+      def assert_response_bad(response)
+        unless assertion_negated( [400].include?(response.code) )
+          raise AssertionException.new "Bad response code: expected 400, but found #{response.code}", response.body
+        end
+      end
+
+      def assert_navigation_links(bundle)
+        unless assertion_negated( bundle.first_link && bundle.last_link && bundle.next_link )
+          raise AssertionException.new "Expecting first, next and last link to be present"
+        end
+      end
+
+      def assert_bundle_response(response)
+        unless assertion_negated( response.resource.class == FHIR::Bundle )
+          # check what this is...
+          found = response.resource
+          begin
+            found = FHIR::Resource.from_contents(response.body)
+          rescue
+            found = nil
+          end
+          raise AssertionException.new "Expected FHIR Bundle but found: #{found.class.name.demodulize}", response.body
+        end
+      end
+
+      def assert_bundle_entry_count(response, count)
+        unless assertion_negated( response.resource.total == count.to_i )
+          raise AssertionException.new "Expected FHIR Bundle with #{count} entries but found: #{response.resource.total} entries", response.body
+        end
+      end
+
+      def assert_bundle_transactions_okay(response)
+        response.resource.entry.each do |entry|
+          unless assertion_negated( !entry.response.nil? )
+            raise AssertionException.new "All Transaction/Batch Bundle.entry elements SHALL have a response." 
+          end
+          status = entry.response.status
+          unless assertion_negated( status && status.start_with?('200','201','204') )
+            raise AssertionException.new "Expected all Bundle.entry.response.status to be 200, 201, or 204; but found: #{status}"
+          end
+        end
+      end
+
+      def assert_resource_content_type(client_reply, content_type)
+        header = client_reply.response[:headers]['content-type']
+        response_content_type = header
+        response_content_type = header[0, header.index(';')] if !header.index(';').nil?
+
+        unless assertion_negated( "application/#{content_type}+fhir" == response_content_type )
+          raise AssertionException.new "Expected a #{content_type} content-type", response_content_type
+        end
+
+      end
+
+      # Based on MIME Types defined in
+      # http://hl7.org/fhir/2015May/http.html#2.1.0.6
+      def assert_valid_resource_content_type_present(client_reply)
+        header = client_reply.response[:headers]['content-type']
+        content_type = header
+        charset = encoding = nil
+
+        content_type = header[0, header.index(';')] if !header.index(';').nil?
+        charset = header[header.index('charset=')+8..-1] if !header.index('charset=').nil?
+        encoding = Encoding.find(charset) if !charset.nil?
+
+        unless assertion_negated( encoding == Encoding::UTF_8 )
+          raise AssertionException.new "Response content-type specifies encoding other than UTF-8: #{charset}", header
+        end
+        unless assertion_negated( (content_type == FHIR::Formats::ResourceFormat::RESOURCE_XML) || (content_type == FHIR::Formats::ResourceFormat::RESOURCE_JSON) )
+          raise AssertionException.new "Invalid FHIR content-type: #{content_type}", header
+        end
+      end
+
+      def assert_etag_present(client_reply)
+        header = client_reply.response[:headers]['etag']
+        assert assertion_negated( !header.nil? ), 'ETag HTTP header is missing.'
+      end
+
+      def assert_last_modified_present(client_reply)
+        header = client_reply.response[:headers]['last-modified']
+        assert assertion_negated( !header.nil? ), 'Last-modified HTTP header is missing.'
+      end
+
+      def assert_valid_content_location_present(client_reply)
+        header = client_reply.response[:headers]['location']
+        assert assertion_negated( !header.nil? ), 'Location HTTP header is missing.'
+      end
+
+      def assert_response_code(response, code)
+        unless assertion_negated( code.to_s == response.code.to_s )
+          raise AssertionException.new "Bad response code: expected #{code}, but found #{response.code}", response.body
+        end
+      end
+
+      def assert_resource_type(response, resource_type)
+        unless assertion_negated( !response.resource.nil? && response.resource.class == resource_type )
+          raise AssertionException.new "Bad response type: expected #{resource_type}, but found #{response.resource.class}.", response.body
+        end
+      end
+
+      def assert_minimum(response, fixture)
+        resource_xml = response.try(:resource).try(:to_xml) || response.try(:body)
+        fixture_xml = fixture.try(:to_xml)
+
+        resource_doc = Nokogiri::XML(resource_xml)
+        raise "Could not retrieve Resource as XML from response" if resource_doc.root.nil?
+        resource_doc.root.add_namespace_definition('fhir', 'http://hl7.org/fhir')
+
+        fixture_doc = Nokogiri::XML(fixture_xml)
+        raise "Could not retrieve Resource as XML from fixture" if fixture_doc.root.nil?
+        fixture_doc.root.add_namespace_definition('fhir', 'http://hl7.org/fhir')
+
+        # FIXME: This doesn't seem to work for a simple case...needs more work!
+        # diffs = []
+        # d1 = Nokogiri::XML('<b><p><a>1</a><b>2</b></p><p><a>2</a><b>3</b></p></b>')
+        # d2 = Nokogiri::XML('<p><a>2</a><b>3</b></p>')
+        # d2.diff(d1, :removed=>true){|change, node| diffs << node.to_xml}
+        # diffs.empty? # this returns a list with d2 in it...
+
+        diffs = []
+        fixture_doc.diff(resource_doc, :removed => true){|change, node| diffs << node.to_xml}
+        diffs.select!{|d| d.strip.length > 0}
+
+        unless assertion_negated( diffs.empty? )
+          raise AssertionException.new "Found #{diffs.length} difference(s) between minimum and actual resource.", diffs.to_s
+        end
+      end
+
+      def assertion_negated(expression)
+        if @negated then !expression else expression end
+      end
+
+      def skip
+        raise SkipException.new
+      end
+
+    end
+
+    class AssertionException < Exception
+      attr_accessor :data
+      def initialize(message, data=nil)
+        super(message)
+        @data = data
+      end
+    end
+
+    class SkipException < Exception
+    end
+
+  end
+end

data/lib/tests/base_test.rb

@@ -0,0 +1,117 @@
+module Crucible
+  module Tests
+    class BaseTest
+
+      include Crucible::Tests::Assertions
+
+      BASE_SPEC_LINK = 'http://hl7.org/fhir/DSTU2'
+      REST_SPEC_LINK = "#{BASE_SPEC_LINK}/http.html"
+
+      attr_accessor :tests_subset
+      attr_accessor :tags
+      attr_accessor :category
+
+      # Base test fields, used in Crucible::Tests::Executor.list_all
+      JSON_FIELDS = ['author','description','id','tests','title', 'multiserver', 'tags', 'details', 'category']
+      STATUS = {
+        pass: 'pass',
+        fail: 'fail',
+        error: 'error',
+        skip: 'skip'
+      }
+      METADATA_FIELDS = ['links', 'requires', 'validates']
+
+      def initialize(client, client2=nil)
+        @client = client
+        @client2 = client2
+        @client.monitor_requests if @client
+        @client2.monitor_requests if @client2
+        @tags ||= []
+      end
+
+      def multiserver
+        false
+      end
+
+      def execute
+        @client.use_format_param = false if @client
+        @client2.use_format_param = false if @client2
+        {id => execute_test_methods}
+      end
+
+      def requires_authorization
+        true
+      end
+
+      def execute_test_methods
+        result = []
+        begin
+          setup if respond_to? :setup and not @metadata_only
+        rescue AssertionException => e
+          @setup_failed = e
+        end
+        prefix = if @metadata_only then 'generating metadata' else 'executing' end
+        methods = tests
+        methods = tests & @tests_subset unless @tests_subset.blank?
+        methods.each do |test_method|
+          @client.requests = [] if @client
+          puts "[#{title}#{('_' + @resource_class.name.demodulize) if @resource_class}] #{prefix}: #{test_method}..."
+          begin
+            result << execute_test_method(test_method)
+          rescue => e
+            result << TestResult.new('ERROR', "Error #{prefix} #{test_method}", STATUS[:error], "#{test_method} failed, fatal error: #{e.message}", e.backtrace.join("\n")).to_hash.merge!({:test_method => test_method})
+          end
+        end
+        teardown if respond_to? :teardown and not @metadata_only
+        result
+      end
+
+      def execute_test_method(test_method)
+        response = self.method(test_method).call().to_hash.merge!({:test_method => test_method })
+        response.merge!({:requests => @client.requests.map { |r| r.to_hash } }) if @client
+        response
+      end
+
+      def author
+        # String identifying test file author
+        self.class.name
+      end
+
+      def description
+        # String containing test file description
+        self.class.name
+      end
+
+      def details
+        {}
+      end
+
+      def id
+        # String used to order test files for execution
+        self.object_id.to_s
+      end
+
+      def tests(keys=nil)
+        # Array of test methods within test file
+        methods = self.methods.grep(/_test$/)
+        if keys
+          matches = []
+          keys.each do |key|
+            matches << methods.grep(/^#{key}/i)
+          end
+          methods = matches.flatten
+        end
+        methods
+      end
+
+      def warning
+        begin
+          yield
+        rescue AssertionException => e
+          @warnings << e.message
+        end
+      end
+
+    end
+  end
+end

data/lib/tests/suites/argonaut_resprint_1_test.rb

@@ -0,0 +1,260 @@
+module Crucible
+  module Tests
+    class ArgonautResprint1Test < BaseSuite
+      attr_accessor :rc
+      attr_accessor :conformance
+      attr_accessor :searchParams
+      attr_reader   :canSearchById
+      attr_accessor :patient_id
+
+      def id
+        'ArgonautResprint1Test'
+      end
+
+      def description
+        'In Re-Sprint 1, we\'ll get up to speed on Argonaut\'s updated implementation guidance for: Patient, Allergies, and Problems & Health Concerns.'
+      end
+
+      def details
+        {
+          'Overview' => 'Since the Argonaut Implementation Program began in 2015, we\'ve come a long way. We\'ve gained early implementation experience working with FHIR DSTU2 and the Data Access Framework profiles — and we\'ve produced updated guidance based on this experience. We\'re running a series of "Re-Sprints" with three goals: ensure we have a chance to battle-test our latest "best practices" in time for MU3; help existing Argonaut implementers come up to speed; and provide an easy on-ramp for new Argonaut implementers.',
+          'Instructions' => 'If you\'re working on a server, please complete the "servers" tab of the Sprints Spreadsheet. You\'ll need to update the status flag to indicate whether you\'ve begun work (or completed work), so clients will know when to start testing. You\'ll also share details about how a developer can obtain OAuth client credentials (client_id for public apps, or a client_id and client_secret for confidential apps) as well as user login credentials. You might consider simply sharing a set of fixed credentials in this spreadsheet, or else directing users to a web page where they can complete self-service registration. If absolutely necessary, you can ask developers to e-mail you directly. If you\'re working on a client, please complete the "clients" tab of the Sprints Spreadsheet. You\'ll also need to update the status flag to indicate whether you\'ve begun work (or completed work).',
+          'FHIR API Calls' => 'For this sprint, EHRs should focus on the following FHIR Resources: Patient, AllergyIntolerance, and Condition. Patients should be retrieveable by ID, and should be searchable by demographics such as name, gender and birthdate. AllergyIntolerances should be searchable by the associated Patient ID. Conditions should be retrievable by code, where code is in the Problem valueset, and should be searchable by Patient ID as well.'
+        }
+      end
+
+      def initialize(client1, client2=nil)
+        super(client1, client2)
+        @rc = FHIR::Patient
+        @tags.append('argonaut')
+        @category = {id: 'argonaut', title: 'Argonaut'}
+        @status_codes = ['active', 'unconfirmed', 'confirmed', 'inactive', 'resolved', 'refuted', 'entered-in-error']
+      end
+
+      def setup
+        if !@client.client.try(:params).nil? && @client.client.params['patient']
+          @patient_id = @client.client.params['patient']
+        end
+      end
+
+      test 'ARS101', 'Get patient by ID' do
+        metadata {
+          links "#{REST_SPEC_LINK}#read"
+          requires resource: "Patient", methods: ["read", "search"]
+          validates resource: "Patient", methods: ["read", "search"]
+        }
+        
+        begin
+          options = {
+            :search => {
+              :flag => true,
+              :compartment => nil,
+              :parameters => {
+                _count: 1
+              }
+            }
+          }
+          @patient_id ||= @client.search(@rc, options).resource.entry.first.resource.xmlId
+        rescue NoMethodError
+          @patient = nil
+        end
+
+        skip if !@patient_id
+
+        reply = @client.read(FHIR::Patient, @patient_id)
+        assert_response_ok(reply)
+        assert_equal @patient_id, reply.id, 'Server returned wrong patient.'
+        @patient = reply.resource
+        assert @patient, "could not get patient by id: #{@patient_id}"
+        warning { assert_valid_resource_content_type_present(reply) }
+        warning { assert_last_modified_present(reply) }
+      end
+
+      test 'ARS102', 'Search by identifier' do
+        metadata {
+          define_metadata('search')
+        }
+        skip if !@patient
+        get_patient_by_param(:identifier => @patient[:identifier].first.try(:value))
+      end
+
+      test 'ARS103', 'Identifier without search keyword' do
+        metadata {
+          define_metadata('search')
+        }
+        skip if !@patient
+        get_patient_by_param({ :identifier => @patient[:identifier].first.try(:value) }, false)
+      end
+
+      test 'ARS104', 'Search by Family & Given' do
+        metadata {
+          define_metadata('search')
+        }
+        skip if !@patient
+        family = @patient[:name].first.try(:family).try(:first)
+        given = @patient[:name].first.try(:given).try(:first)
+        assert family, "Patient family name not returned"
+        assert given, "Patient given name not returned"
+        get_patient_by_param(family: family, given: given)
+      end
+
+      test 'ARS105', 'Family & Given without search keyword' do
+        metadata {
+          define_metadata('search')
+        }
+        skip if !@patient
+        family = @patient[:name].first.try(:family).try(:first)
+        given = @patient[:name].first.try(:given).try(:first)
+        assert family, "Patient family name not provided"
+        assert given, "Patient given name not provided"
+        get_patient_by_param({ family: family, given: given }, false)
+      end
+
+      test 'ARS106', 'Search by name and gender' do
+        metadata {
+          define_metadata('search')
+        }
+        skip if !@patient
+        name = @patient[:name].first.try(:family).try(:first)
+        gender = @patient[:gender]
+        assert name, "Patient name not provided"
+        assert gender, "Patient gender not provided"
+        get_patient_by_param(name: name, gender: gender)
+      end
+
+      test 'ARS107', 'Name and gender without search keyword' do
+        metadata {
+          define_metadata('search')
+        }
+        skip if !@patient
+        name = @patient[:name].first.try(:family).try(:first)
+        gender = @patient[:gender]
+        assert name, "Patient name not provided"
+        assert gender, "Patient gender not provided"
+        get_patient_by_param(name: name, gender: gender)
+      end
+
+      test 'ARS108', 'Search by Birthdate' do
+        metadata {
+          define_metadata('search')
+        }
+        skip if !@patient
+        birthdate = @patient[:birthDate]
+        gender = @patient[:gender]
+        assert birthdate, "Patient birthdate not provided"
+        assert gender, "Patient gender not provided"
+        get_patient_by_param(birthdate: birthdate, gender: gender)
+      end
+
+      test 'ARS109', 'Birthdate without search keyword' do
+        metadata {
+          define_metadata('search')
+        }
+        skip if !@patient
+        birthdate = @patient[:birthDate]
+        gender = @patient[:gender]
+        assert birthdate, "Patient birthdate not provided"
+        assert gender, "Patient gender not provided"
+        get_patient_by_param({ birthdate: birthdate, gender: gender }, false)
+      end
+
+      test 'ARS110', 'GET AllergyIntolerance with Patient IDs' do
+        metadata {
+          links "#{REST_SPEC_LINK}#search"
+          requires resource: 'Patient', methods: ['read', 'search']
+          requires resource: 'AllergyIntolerance', methods: ['read', 'search']
+          validates resource: 'Patient', methods: ['read', 'search']
+          validates resource: 'AllergyIntolerance', methods: ['read', 'search']
+        }
+
+        assert !@client.client.try(:params).nil?, "The client was not authorized for the test"
+        assert @client.client.params["patient"], "No patient parameter was passed to the client"
+
+        patient_id = @patient.xmlId
+
+        options = {
+          search: {
+            flag: false,
+            compartment: nil,
+            parameters: {
+              patient: patient_id
+            }
+          }
+        }
+
+        reply = @client.search(FHIR::AllergyIntolerance, options)
+
+        assert_response_ok(reply)
+
+        reply.resource.entry.each do |entry|
+          assert (entry.resource.patient && entry.resource.patient.reference.include?(patient_id)), "Patient on AllergyIntolerance does not match patient requested"
+          assert entry.resource.substance, "No substance defined for AllergyIntolerance"
+          entry.resource.substance.coding.each do |coding|
+            warn { assert ['http://rxnav.nlm.nih.gov/REST/Ndfrt', 'http://snomed.info/sct', 'http://www.nlm.nih.gov/research/umls/rxnorm'].include?(coding.system), "Code system #{coding.system} does not match expected code system for Allergy substance coding" }
+          end
+          assert @status_codes.include?(entry.resource.status), 'Allergy Status is not part of status Value Set'
+        end
+      end
+
+      test 'ARS111', 'GET Condition with Patient IDs' do
+        metadata {
+          links "#{REST_SPEC_LINK}#search"
+          requires resource: "Patient", methods: ["read", "search"]
+          validates resource: "Patient", methods: ["read", "search"]
+        }
+
+        assert !@client.client.try(:params).nil?, "The client was not authorized for the test"
+        assert @client.client.params["patient"], "No patient parameter was passed to the client"
+
+        patient_id = @patient.xmlId
+
+        options = {
+          search: {
+            flag: false,
+            compartment: nil,
+            parameters: {
+              patient: patient_id
+            }
+          }
+        }
+
+        reply = @client.search(FHIR::Condition, options)
+
+        assert_response_ok(reply)
+
+        reply.resource.entry.each do |entry|
+          assert (entry.resource.patient && entry.resource.patient.reference.include?(patient_id)), "Patient on condition does not match patient requested"
+          entry.resource.code.coding.each do |coding|
+            warn { assert coding.system == "http://snomed.info/sct", "Condition Code System is not SNOMED-CT" }
+            assert !coding.code.empty?, "No code defined for Coding"
+          end
+        end
+      end
+
+      private
+
+      def get_patient_by_param(params = {}, flag = true)
+        assert !params.empty?, "No params for patient"
+        options = {
+          :search => {
+            :flag => flag,
+            :compartment => nil,
+            :parameters => params
+          }
+        }
+        reply = @client.search(@rc, options)
+        assert_response_ok(reply)
+        assert_bundle_response(reply)
+        assert reply.resource.get_by_id(@patient_id).equals?(@patient, ['_id', "text", "meta", "lastUpdated"]), 'Server returned wrong patient.'
+      end
+
+      def define_metadata(method)
+        links "#{REST_SPEC_LINK}##{method}"
+        links "#{BASE_SPEC_LINK}/#{@rc.name.demodulize.downcase}.html"
+        requires resource: @rc.name.demodulize, methods: [method]
+        validates resource: @rc.name.demodulize, methods: [method]
+      end
+
+    end
+  end
+end