health-data-standards 3.5.3 → 3.6.1

data/lib/health-data-standards/validate/measure_validator.rb

@@ -0,0 +1,127 @@
+module HealthDataStandards
+  module Validate
+    class MeasureValidator
+      include BaseValidator
+
+      def initialize(template_oid)
+        @template_oid = template_oid
+
+      end
+
+      def validate(file, data={})
+	@errors = []
+	@doc = get_document(file)
+	@doc.root.add_namespace_definition('cda', 'urn:hl7-org:v3')
+	measure_ids = HealthDataStandards::CQM::Measure.all.map(&:hqmf_id)
+	doc_measure_ids = @doc.xpath(measure_selector).map(&:value).map(&:upcase)
+	#list of all of the set ids in the QRDA
+	doc_neutral_ids = @doc.xpath(neutral_measure_selector).map(&:value).map(&:upcase).sort
+	#list of all of the setids in the QRDA that are also in the bundle, includes duplicates if code appears twice in document
+	bundle_neutral_ids = HealthDataStandards::CQM::Measure.distinct(:hqmf_set_id)
+	doc_bundle_neutral_ids = doc_neutral_ids - (doc_neutral_ids - bundle_neutral_ids)
+	validate_measure_ids(doc_measure_ids, measure_ids, data)
+	validate_set_ids(doc_neutral_ids, doc_bundle_neutral_ids, data)
+	if validate_no_repeating_measure_population_ids(data)
+		validate_measure_ids_set_ids_usage(doc_bundle_neutral_ids, doc_measure_ids, data)
+	end
+
+	@errors
+      end
+
+      private
+
+		#returns true if there are no repeating measures, check to see that the measure id usage is correct
+		def validate_no_repeating_measure_population_ids(data={})
+		  noDuplicateMeasures = true
+	      doc_population_ids = @doc.xpath(measure_population_selector).map(&:value).map(&:upcase).sort
+	      duplicates = doc_population_ids.group_by{ |e| e }.select { |k, v| v.size > 1 }.map(&:first)
+	      duplicates.each do |duplicate|
+	        measureId = @doc.xpath(find_measure_node_for_population(duplicate)).at_xpath("cda:reference/cda:externalDocument/cda:id[./@root='2.16.840.1.113883.4.738']/@extension")
+	        @errors << build_error("Population #{duplicate} for Measure #{measureId.value} reported more than once", "/", data[:file_name])
+	        noDuplicateMeasures = false
+	      end
+	      return noDuplicateMeasures
+	  end
+
+      def validate_measure_ids(doc_measure_ids, measure_ids, data={})
+	(doc_measure_ids - measure_ids).map do |hqmf_id|
+	  @errors << build_error("Invalid HQMF ID Found: #{hqmf_id}", "/", data[:file_name])
+	end
+      end
+
+      def validate_set_ids(doc_neutral_ids, doc_bundle_neutral_ids, data={})
+	#an error will be returned for all of the setids that are in the QRDA that aren't in the bundle
+	(doc_neutral_ids - doc_bundle_neutral_ids).map do |hqmf_set_id|
+	  @errors << build_error("Invalid HQMF Set ID Found: #{hqmf_set_id}", "/", data[:file_name])
+	end
+      end
+
+	#does not work if the same measure is reported more than once, nested under the repeating measures test
+      def validate_measure_ids_set_ids_usage(doc_bundle_neutral_ids, doc_measure_ids, data={})
+	#for each of the setIds that are in the bundle, check that they are for the correct measure id
+	entries_start_position = @doc.xpath(first_entry)
+	previous = ""
+	index = 1
+	doc_bundle_neutral_ids.each do |hqmf_set_id|
+	  #selects the measure id that is in the same entry as the set id
+	  #iterates through multiple instances of the same setId
+	  if previous == hqmf_set_id
+	    index = index + 1
+      else
+        index = 1
+	  end
+	  measure_id_entry = doc_measure_ids[(@doc.xpath(location_of_set_id(hqmf_set_id,index)) - entries_start_position)]
+	  previous = hqmf_set_id
+	  #queries database to see if there is a measure with the combindation of setId and measureId
+	  if HealthDataStandards::CQM::Measure.where(hqmf_id: measure_id_entry, hqmf_set_id: hqmf_set_id).length() == 0
+	    @errors << build_error("Invalid HQMF Set ID Found: #{hqmf_set_id} for HQMF ID: #{measure_id_entry}", "/", data[:file_name])
+	  end
+	end
+      end
+
+      def measure_selector
+	"/cda:ClinicalDocument/cda:component/cda:structuredBody/cda:component/cda:section/cda:entry" +
+	"/cda:organizer[./cda:templateId[@root='#{@template_oid}']]/cda:reference[@typeCode='REFR']" +
+        "/cda:externalDocument[@classCode='DOC']/cda:id[@root='2.16.840.1.113883.4.738']/@extension"
+      end
+
+      #finds all of the setIds in the QRDA document
+      def neutral_measure_selector
+	"/cda:ClinicalDocument/cda:component/cda:structuredBody/cda:component/cda:section/cda:entry" +
+	"/cda:organizer[./cda:templateId[@root='#{@template_oid}']]/cda:reference[@typeCode='REFR']" +
+	"/cda:externalDocument[@classCode='DOC']/cda:setId/@root"
+      end
+
+      #finds the node index of the first entry element in the measure template
+      def first_entry
+	"count(//cda:entry[cda:organizer[./cda:templateId[@root='#{@template_oid}']]" +
+	"/cda:reference[@typeCode='REFR']/cda:externalDocument[@classCode='DOC']" +
+	"/cda:id[@root='2.16.840.1.113883.4.738']][1]/preceding-sibling::*)+1"
+      end
+
+      #finds the node index of the extry that the specified setId is in, index is used if the same setId appears twice
+      def location_of_set_id(set_id,index)
+	"count(//cda:entry[cda:organizer[./cda:templateId[@root='#{@template_oid}']]" +
+	"/cda:reference[@typeCode='REFR']/cda:externalDocument[@classCode='DOC']" +
+    "/cda:setId[@root[contains(translate(.,'abcdefghijklmnopqrstuvwxyz','ABCDEFGHIJKLOMNOPQRSTUVWXYZ')" +
+    ",'#{set_id}')]]][#{index}]/preceding-sibling::*)+1"
+      end
+
+	  
+	  def measure_population_selector
+	  "/cda:ClinicalDocument/cda:component/cda:structuredBody/cda:component/cda:section/cda:entry" +
+	  "/cda:organizer[./cda:templateId[@root='2.16.840.1.113883.10.20.27.3.1']]/cda:component" +
+	  "/cda:observation[./cda:templateId[@root='2.16.840.1.113883.10.20.27.3.5']]/cda:reference" + 
+	  "/cda:externalObservation/cda:id/@root"
+	  end
+
+	  def find_measure_node_for_population(id)
+	  "/cda:ClinicalDocument/cda:component/cda:structuredBody/cda:component/cda:section/cda:entry" +
+	  "/cda:organizer[ ./cda:templateId[@root='2.16.840.1.113883.10.20.27.3.1']" +
+	  "and ./cda:component/cda:observation[./cda:templateId[@root='2.16.840.1.113883.10.20.27.3.5']]/cda:reference" +
+	  "/cda:externalObservation/cda:id[@root='#{id.upcase}']]"
+	  end
+    end
+  end
+
+end

data/lib/health-data-standards/validate/performance_rate_validator.rb

@@ -0,0 +1,94 @@
+require_relative "reported_result_extractor"
+module HealthDataStandards
+  module Validate
+  class PerformanceRateValidator
+    include ReportedResultExtractor
+    include BaseValidator
+    
+
+    def initialize()
+
+
+    
+    end
+
+    # Nothing to see here - Move along
+    def validate(file, data = {})
+      errorsList = []
+      document = get_document(file)
+      #grab measure IDs from QRDA file
+      measure_ids = document.xpath(measure_selector).map(&:value).map(&:upcase)
+      measure_ids.each do |measure_id|
+        measures = HealthDataStandards::CQM::Measure.where(id: measure_id)
+        measures.each do |measure|
+          result_key = measure["population_ids"].dup
+          reported_result, errors = extract_results_by_ids(measure['id'], result_key, document)
+          #only check performace rate when there is one
+          if reported_result['PR'] != nil
+            error = check_performance_rates(reported_result, result_key, measure['id'], data)
+            if error != nil
+              errorsList << error
+            end
+          end
+        end
+      end
+      errorsList
+    end
+
+    def calculate_performance_rates(reported_result)
+      #Just in case a measure does not report these populations
+      denex = 0
+      denexcep = 0
+      denom = 0
+      numer = 0
+      if reported_result['DENEX'] != nil
+        denex = reported_result['DENEX']
+      end
+      if reported_result['DENEXCEP'] != nil
+        denexcep = reported_result['DENEXCEP']
+      end
+      if reported_result['DENOM'] != nil
+        denom = reported_result['DENOM']
+      end
+      if reported_result['NUMER'] != nil
+        numer = reported_result['NUMER']
+      end
+      denom = denom -  denex - denexcep
+      pr = 0
+      if denom == 0
+        pr = "NA"
+      else
+        pr = numer / denom.to_f
+      end
+      return pr
+    end
+
+    def check_performance_rates(reported_result, population_ids, measure_id, data = {})
+      expected = calculate_performance_rates(reported_result)
+      _ids = population_ids
+      if expected == "NA"
+        if reported_result['PR']['nullFlavor'] != "NA"
+          return build_error("Reported Performance Rate for Numerator #{_ids['NUMER']} should be NA", "/", data[:file_name])
+        end
+      else
+        if reported_result['PR']['nullFlavor'] == "NA"
+          return build_error("Reported Performance Rate for Numerator #{_ids['NUMER']} should not be NA", "/", data[:file_name])
+        else
+          if (reported_result['PR']['value'].split('.',2).last.size > 6)
+            return build_error("Reported Performance Rate SHALL not have a precision greater than .000001 ", "/", data[:file_name])
+          elsif (reported_result['PR']['value'].to_f - expected.round(6)).abs > 0.0000001
+            return build_error("Reported Performance Rate of #{reported_result['PR']['value']} for Numerator #{_ids['NUMER']} does not match expected value of #{expected.to_s[0,8]}.", "/", data[:file_name])
+          end
+        end
+      end
+    end
+
+    def measure_selector
+  "/cda:ClinicalDocument/cda:component/cda:structuredBody/cda:component/cda:section/cda:entry" +
+  "/cda:organizer[./cda:templateId[@root='2.16.840.1.113883.10.20.27.3.1']]/cda:reference[@typeCode='REFR']" +
+        "/cda:externalDocument[@classCode='DOC']/cda:id[@root='2.16.840.1.113883.4.738']/@extension"
+    end
+
+    end
+  end
+end

data/lib/health-data-standards/validate/reported_result_extractor.rb

@@ -0,0 +1,170 @@
+module HealthDataStandards
+  module Validate
+module ReportedResultExtractor
+
+    #takes a document and a list of 1 or more id hashes, e.g.:
+      #[{measure_id:"8a4d92b2-36af-5758-0136-ea8c43244986", set_id:"03876d69-085b-415c-ae9d-9924171040c2", ipp:"D77106C4-8ED0-4C5D-B29E-13DBF255B9FF", den:"8B0FA80F-8FFE-494C-958A-191C1BB36DBF", num:"9363135E-A816-451F-8022-96CDA7E540DD"}]
+      #returns nil if nothing matching is found
+      # returns a hash with the values of the populations filled out along with the population_ids added to the result
+
+
+      def extract_results_by_ids(measure_id, ids, doc)
+        results = nil
+        _ids = ids.dup
+        stratification = _ids.delete("stratification")
+        stratification ||= _ids.delete("STRAT")
+        errors = []
+        nodes = find_measure_node(measure_id, doc)
+
+        if nodes.nil? || nodes.empty?
+          # short circuit and return nil
+          return {}
+        end
+
+        nodes.each do |n|
+         results =  get_measure_components(n, _ids, stratification)
+         break if (results != nil || (results != nil && !results.empty?))
+        end
+        return nil if results.nil?
+        results[:population_ids] = ids.dup
+        results
+      end
+
+      def find_measure_node(id, doc)
+         xpath_measures = %Q{/cda:ClinicalDocument/cda:component/cda:structuredBody/cda:component/cda:section
+          /cda:entry/cda:organizer[ ./cda:templateId[@root = "2.16.840.1.113883.10.20.27.3.1"]
+          and ./cda:reference/cda:externalDocument/cda:id[#{translate("@extension")}='#{id.upcase}' and #{translate("@root")}='2.16.840.1.113883.4.738']]}
+         return doc.xpath(xpath_measures)
+      end
+
+      def get_measure_components(n,ids, stratification)
+        results = {:supplemental_data =>{}}
+        ids.each_pair do |k,v|
+          val = nil
+          sup = nil
+          if (k == 'OBSERV')
+            msrpopl = ids['MSRPOPL']
+            val, sup = extract_cv_value(n,v,msrpopl, stratification)
+          else
+            val,sup,pr =extract_component_value(n,k,v,stratification)
+          end
+          if !val.nil?
+            results[k.to_s] = val
+            results[:supplemental_data][k] = sup
+          else
+            # return nil
+          end
+          if !pr.nil?
+            results["PR"] = pr
+          end
+        end
+        results
+      end
+
+    def extract_cv_value(node, id, msrpopl, strata = nil)
+        xpath_observation = %{ cda:component/cda:observation[./cda:value[@code = "MSRPOPL"] and ./cda:reference/cda:externalObservation/cda:id[#{translate("@root")}='#{msrpopl.upcase}']]}
+        cv = node.at_xpath(xpath_observation)
+        return nil unless cv
+        val = nil
+        if strata
+        strata_path = %{ cda:entryRelationship[@typeCode="COMP"]/cda:observation[./cda:templateId[@root = "2.16.840.1.113883.10.20.27.3.4"]  and ./cda:reference/cda:externalObservation/cda:id[#{translate("@root")}='#{strata.upcase}']]}
+        n = cv.xpath(strata_path)
+        val = get_cv_value(n,id)
+        else
+        val = get_cv_value(cv,id)
+        end
+        return val, (strata.nil? ?  extract_supplemental_data(cv) : nil)
+      end
+
+      def extract_component_value(node, code, id, strata = nil)
+        xpath_observation = %{ cda:component/cda:observation[./cda:value[@code = "#{code}"] and ./cda:reference/cda:externalObservation/cda:id[#{translate("@root")}='#{id.upcase}']]}
+        cv = node.at_xpath(xpath_observation)
+        return nil unless cv
+        val = nil
+        if strata
+          strata_path = %{ cda:entryRelationship[@typeCode="COMP"]/cda:observation[./cda:templateId[@root = "2.16.840.1.113883.10.20.27.3.4"]  and ./cda:reference/cda:externalObservation/cda:id[#{translate("@root")}='#{strata.upcase}']]}
+          n = cv.xpath(strata_path)
+          val = get_aggregate_count(n) if n
+        else
+          val = get_aggregate_count(cv)
+        end
+        #Performance rate is only applicable for unstratified values
+        if code == "NUMER" && strata == nil
+          pref_rate_value = extract_performance_rate(node,code,id)
+        end
+        return val,(strata.nil? ?  extract_supplemental_data(cv) : nil),pref_rate_value
+      end
+
+    def extract_performance_rate(node,code,id)
+        xpath_perf_rate = %{ cda:component/cda:observation[./cda:templateId[@root = "2.16.840.1.113883.10.20.27.3.14"] and ./cda:reference/cda:externalObservation/cda:id[#{translate("@root")}='#{id.upcase}']]/cda:value}
+        perf_rate = node.at_xpath(xpath_perf_rate)
+        pref_rate_value = {}
+        if perf_rate != nil
+          if perf_rate.at_xpath("./@nullFlavor")
+            pref_rate_value["nullFlavor"] = "NA"
+            return pref_rate_value
+          else
+            pref_rate_value["value"] = perf_rate.at_xpath("./@value").value
+            return pref_rate_value
+          end
+        end
+        return nil
+      end
+          # convert numbers in value nodes to Int / Float as necessary TODO add more types other than 'REAL'
+      def convert_value(value_node)
+        if value_node.nil?
+          return
+        end
+        if value_node['type'] == 'REAL' || value_node['value'].include?('.')
+          return value_node['value'].to_f
+        else
+          return value_node['value'].to_i
+        end
+      end
+
+      #given an observation node with an aggregate count node, return the reported and expected value within the count node
+      def get_cv_value(node, cv_id)
+        xpath_value = %{cda:entryRelationship/cda:observation[./cda:templateId[@root="2.16.840.1.113883.10.20.27.3.2"] and ./cda:reference/cda:externalObservation/cda:id[#{translate("@root")}='#{cv_id.upcase}']]/cda:value}
+
+        value_node = node.at_xpath(xpath_value)
+        value = convert_value(value_node) if value_node
+        value
+      end
+
+      #given an observation node with an aggregate count node, return the reported and expected value within the count node
+      def get_aggregate_count(node)
+        xpath_value = 'cda:entryRelationship/cda:observation[./cda:templateId[@root="2.16.840.1.113883.10.20.27.3.3"]]/cda:value'
+        value_node = node.at_xpath(xpath_value)
+        value = convert_value(value_node) if value_node
+        value
+      end
+
+      def extract_supplemental_data(cv)
+        ret = {}
+        supplemental_data_mapping = {"RACE"=> "2.16.840.1.113883.10.20.27.3.8",
+                             "ETHNICITY" => "2.16.840.1.113883.10.20.27.3.7",
+                             "SEX" => "2.16.840.1.113883.10.20.27.3.6",
+                             "PAYER" => "2.16.840.1.113883.10.20.27.3.9"}
+        supplemental_data_mapping.each_pair do |supp, id|
+          key_hash = {}
+          xpath = "cda:entryRelationship/cda:observation[cda:templateId[@root='#{id}']]"
+          (cv.xpath(xpath) || []).each do |node|
+            value = node.at_xpath('cda:value')
+            count = get_aggregate_count(node)
+            if value.at_xpath("./@nullFlavor")
+             key_hash["UNK"] = count
+            else
+             key_hash[value['code']] = count
+            end
+          end
+          ret[supp.to_s] = key_hash
+        end
+        ret
+      end
+
+      def translate(id)
+        %{translate(#{id}, "abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZ")}
+      end
+    end
+end
+end

data/lib/health-data-standards/validate/schema_validator.rb

@@ -0,0 +1,24 @@
+module HealthDataStandards
+  module Validate
+    module Schema
+      class Validator
+        include BaseValidator
+
+        def initialize(name, schema_file)
+          @name = name
+          @schema_file = schema_file
+          @xsd = Nokogiri::XML::Schema(File.new(@schema_file))
+        end
+
+        # Validate the document against the configured schema
+        def validate(document,data={})
+          @xsd.errors.clear
+          doc = get_document(document)
+          @xsd.validate(doc).map do |error|
+            build_error(error.message, "/", data[:file_name])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/health-data-standards/validate/schematron/c_processor.rb

@@ -0,0 +1,28 @@
+module HealthDataStandards
+  module Validate
+    module Schematron
+      module CProcessor
+
+
+        def get_errors(document)
+          document = get_document(document)
+          processor.transform(document)
+        end
+
+        def processor
+          return @processor if @processor
+          doc = Nokogiri::XML(File.open(@schematron_file))
+          doc.root["defaultPhase"] = ("errors")
+
+          xslt = Nokogiri::XSLT(File.open(ISO_SCHEMATRON))
+
+          result = xslt.transform(doc)
+          #this is stupid but needs to be done to assocaite the xslt file with a dirctory
+          result = Nokogiri::XML(result.to_s,@schematron_file)
+          @processor = Nokogiri::XSLT::Stylesheet.parse_stylesheet_doc(result)
+        end
+
+      end
+    end
+  end
+end

data/lib/health-data-standards/validate/schematron/java_processor.rb

@@ -0,0 +1,93 @@
+require 'java'
+require_relative '../../../../java/saxon9he.jar'
+
+java_import "java.io.StringReader"
+java_import "java.io.StringWriter"
+
+java_import "javax.xml.parsers.DocumentBuilder"
+java_import "javax.xml.parsers.DocumentBuilderFactory"
+
+java_import "javax.xml.transform.TransformerFactory"
+java_import "javax.xml.transform.Transformer"
+java_import "javax.xml.transform.dom.DOMSource"
+java_import "javax.xml.transform.stream.StreamSource"
+java_import "javax.xml.transform.stream.StreamResult"
+
+java_import "org.w3c.dom.Document"
+
+ 
+TRANSFORMER_FACTORY_IMPL = "net.sf.saxon.TransformerFactoryImpl"
+
+module HealthDataStandards
+  module Validate
+    module Schematron
+      module JavaProcessor
+
+        ISO_SCHEMATRON2 = File.join(DIR, 'resources/schematron/iso-schematron-xslt2/iso_svrl_for_xslt2.xsl')
+
+        class HdsUrlResolver
+          include javax.xml.transform.URIResolver
+
+          def initialize(schematron)
+            @file = schematron
+          end
+
+          def resolve(href, base)
+            path = File.join(File.dirname(@file), href)
+            return StreamSource.new(java.io.File.new(path))
+          end
+
+        end
+
+        def get_errors(document)
+          document_j = get_document_j(document)
+          output = build_transformer(StringReader.new(processor), StreamSource.new(document_j), true)
+          Nokogiri::XML(output)
+        end
+
+        def get_document_j(doc)
+          case doc
+          when File
+            java.io.File.new(doc.path)
+          else 
+            StringReader.new(doc.to_s)
+          end
+        end
+
+        def processor
+          @processor ||= build_transformer(java.io.File.new(ISO_SCHEMATRON2), schematron_file)
+        end
+
+        def schematron_file
+          # this allows us to run the validation utility app in jBoss/TorqueBox
+          # for some reason it breaks the first time you call DocumentBuilderFactory,
+          # so the solution is to catch the error and retry
+          # TODO: pull this out when the above is no longer the case.
+          begin
+            dbf = DocumentBuilderFactory.new_instance
+          rescue Exception => ex
+            retry
+          end
+          dbf.setIgnoringElementContentWhitespace(true);
+          db = dbf.new_document_builder
+          document = db.parse(java.io.File.new(@schematron_file))
+          
+          root = document.document_element
+          phase = root.set_attribute("defaultPhase", "errors")
+
+          DOMSource.new(root)
+        end
+
+        def build_transformer(xslt, input_file, url=false)
+          factory = TransformerFactory.newInstance(TRANSFORMER_FACTORY_IMPL, nil)
+          factory.uri_resolver = HdsUrlResolver.new(@schematron_file) if url
+          transformer = factory.new_transformer(StreamSource.new(xslt))
+          sw = StringWriter.new
+          output = StreamResult.new(sw)
+          transformer.transform(input_file, output)
+          sw.to_s
+        end
+      end
+    end
+  end
+end