cqm-parsers 0.2.4 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 5ae0c0edc66dbccfb1f7bff982ed28472220b223
-  data.tar.gz: a244b962a1ba17b061ac9fbacdb1ce27ef3d9ae0
+SHA256:
+  metadata.gz: d83d5f1f9b8c3ee8b00ff2acf4340976cf4055ab43e9ab9cf178d91dc62feea5
+  data.tar.gz: 92584ac7191a821a1cce7fff38805f526838b10555b184b643fcb3b479310b32
 SHA512:
-  metadata.gz: aa00da03f1b87750f4dd0fdaa9a60b67616afba80c41f741a35b74b2a2e989b738220d28c5cbfab6ed9087f3b71da829c9a0c3da50e276588ea648d2490853dc
-  data.tar.gz: 29f3149e4f031808f4e956c9d29584ae1acc9cc60b6798192fd4a880fdf41fad302d6da47a0cf1add996a42fccb802e3f5d844a4421d85bc97ca1ba8beb39f45
+  metadata.gz: 0fb80c3815803eb39df96cfee2dda409e3c8d3b675148d8aee06b5fc081ebe8996e952045cc8c0344204c29d02d449063c01850359b009de53bdae8a7f4a262c
+  data.tar.gz: bcb9fa25b6d7c1ae6f9d31dfacd1a242a945a0b91d2c2e46fe58a6ea4e487dde327eee0d8b88b7ad67a70cdfb1482e50f09adbd924e5f2da3c9deb90f0d0a821

data/Gemfile

@@ -2,17 +2,22 @@ source 'https://rubygems.org'
 
 gemspec :development_group => :test
 
-gem 'cqm-models', '~> 0.8.4'
-gem 'mongoid', '~> 5.0.0'
+gem 'mongoid', '~> 6.4.2'
+
+gem 'cqm-models', '~> 2.0.0'
+# gem 'cqm-models', git: 'https://github.com/projecttacoma/cqm-models.git', branch: 'master'
+# gem 'cqm-models', :path => '../cqm-models'
 
 group :development, :test do
   gem 'bundler-audit'
   gem 'rubocop', '~> 0.52.1', require: false
+  gem 'byebug', '~> 6.0.2',  platforms: [:ruby_20, :ruby_21, :ruby_22, :ruby_23]
+  gem 'pry'
+  gem 'pry-nav'
 end
 
 group :development do
   gem 'rake'
-  gem 'byebug', '~> 6.0.2',  platforms: [:ruby_20, :ruby_21, :ruby_22, :ruby_23]
 end
 
 group :test do
@@ -25,6 +30,5 @@ group :test do
   gem 'minitest', '~> 5.3'
   gem 'minitest-reporters'
   gem 'awesome_print', :require => 'ap'
-  gem 'simplexml_parser', :git => 'https://github.com/projecttacoma/simplexml_parser.git', :branch => 'master'
   gem 'vcr'
 end

data/README.md

@@ -1,13 +1,52 @@
+[![Build Status](https://travis-ci.com/projecttacoma/cqm-parsers.svg?branch=master)](https://travis-ci.com/projecttacoma/cqm-parsers)
 [![codecov](https://codecov.io/gh/projecttacoma/cqm-parsers/branch/master/graph/badge.svg)](https://codecov.io/gh/projecttacoma/cqm-parsers)
+[![Gem Version](https://badge.fury.io/rb/cqm-parsers.svg)](https://badge.fury.io/rb/cqm-parsers)
 
+# cqm-parsers
 
-cqm-parsers
-===========
+This project contains libraries for parsing HQMF documents and parsing MAT packages.
 
-This project contains libraries for parsing HQMF documents.
+## Usage (MAT Package Loading)
 
-License
-=======
+To load measures from a MAT package file into the measure model, use the `Measures::CqlLoader` class. It can be used to create an array of measure models. For a composite measure, the array will contain the component measures and the last element will be the composite measure. For a non-composite measure (most measures), the array will contain one item.
+Example measure loading:
+
+```ruby
+
+# Set the VSACValueSetLoader options; in this example we are fetching a specific profile.
+vsac_options = { profile: 'MU2 Update 2016-04-01' }
+
+# Set the measure details. For defaults, you can just pass in {}.
+measure_details = { 'episode_of_care'=> false }
+
+# Load a MAT package from test fixtures.
+measure_file = File.new File.join('some/path/CMS158_v5_4_Artifacts.zip')
+
+# Initialize a value set loader, in this case we are using the VSACValueSetLoader.
+value_set_loader = Measures::VSACValueSetLoader.new(vsac_options, get_ticket_granting_ticket)
+
+# Initialize the CqlLoader with the needed parameters.
+loader = Measures::CqlLoader.new(measure_file, measure_details, value_set_loader)
+# Build an array of measure models.
+measures = loader.extract_measures
+
+```
+
+Note that a different value set loader could be passed in; for example if you had a file containing value sets you could create a loader that read the value sets from file instead of fetching them from VSAC.
+
+## Running the tests
+
+```bash
+
+bundle exec rake test
+
+```
+
+## Versioning
+
+We use [SemVer](http://semver.org/) for versioning. For the versions available, see [tags on this repository](https://github.com/projecttacoma/cqm-parsers/tags). 
+
+## License
 
 Copyright 2018 The MITRE Corporation
 

data/Rakefile

@@ -1,3 +1,4 @@
+require 'simplecov'
 require 'rake/testtask'
 require 'cane/rake_task'
 

data/lib/ext/data_element.rb

@@ -2,7 +2,7 @@ module QDM
   class DataElement
     def merge!(other)
       # ensure they're the same category (e.g. 'encounter')
-      return unless category == other.category
+      return unless qdmCategory == other.qdmCategory
 
       # ensure they're the same status (e.g. 'performed'), and that they both have a status set (or that they both don't)
       return if respond_to?(:qdmStatus) && !other.respond_to?(:qdmStatus)

data/lib/hqmf-parser.rb

@@ -7,6 +7,8 @@ require 'ostruct'
 require_relative 'util/counter.rb'
 require_relative 'util/code_system_helper'
 require_relative 'util/hqmf_template_helper'
+require_relative 'util/vsac_api'
+require_relative 'util/util'
 
 require_relative 'hqmf-model/utilities.rb'
 
@@ -68,49 +70,15 @@ require_relative 'hqmf-parser/converter/pass2/operator_converter'
 
 require_relative 'hqmf-parser/parser'
 
-require_relative 'qrda-export/helper/code_system_helper.rb'
-require_relative 'qrda-export/helper/date_helper.rb'
-require_relative 'qrda-export/helper/cat_1_view_helper.rb'
-
-require_relative 'qrda-export/catI-r5/qrda1_r5.rb'
-
-require_relative 'qrda-import/entry_package.rb'
-require_relative 'qrda-import/cda_identifier.rb'
-require_relative 'qrda-import/narrative_reference_handler.rb'
-require_relative 'qrda-import/entry_finder.rb'
-
-require_relative 'qrda-import/base-importers/section_importer.rb'
-require_relative 'qrda-import/base-importers/demographics_importer.rb'
-require_relative 'qrda-import/base-importers/medication_importer.rb'
-
-require_relative 'qrda-import/data-element-importers/allergy_intolerance_importer.rb'
-require_relative 'qrda-import/data-element-importers/diagnostic_study_order_importer.rb'
-require_relative 'qrda-import/data-element-importers/intervention_order_importer.rb'
-require_relative 'qrda-import/data-element-importers/encounter_performed_importer.rb'
-require_relative 'qrda-import/data-element-importers/diagnosis_importer.rb'
-require_relative 'qrda-import/data-element-importers/medication_active_importer.rb'
-require_relative 'qrda-import/data-element-importers/medication_order_importer.rb'
-require_relative 'qrda-import/data-element-importers/procedure_performed_importer.rb'
-require_relative 'qrda-import/data-element-importers/physical_exam_performed_importer.rb'
-require_relative 'qrda-import/data-element-importers/laboratory_test_performed_importer.rb'
-require_relative 'qrda-import/data-element-importers/adverse_event_importer.rb'
-require_relative 'qrda-import/data-element-importers/assessment_performed_importer.rb'
-require_relative 'qrda-import/data-element-importers/communication_from_patient_to_provider_importer.rb'
-require_relative 'qrda-import/data-element-importers/communication_from_provider_to_patient_importer.rb'
-require_relative 'qrda-import/data-element-importers/communication_from_provider_to_provider_importer.rb'
-require_relative 'qrda-import/data-element-importers/device_applied_importer.rb'
-require_relative 'qrda-import/data-element-importers/device_order_importer.rb'
-require_relative 'qrda-import/data-element-importers/diagnostic_study_performed_importer.rb'
-require_relative 'qrda-import/data-element-importers/encounter_order_importer.rb'
-require_relative 'qrda-import/data-element-importers/immunization_administered_importer.rb'
-require_relative 'qrda-import/data-element-importers/intervention_performed_importer.rb'
-require_relative 'qrda-import/data-element-importers/laboratory_test_order_importer.rb'
-require_relative 'qrda-import/data-element-importers/medication_administered_importer.rb'
-require_relative 'qrda-import/data-element-importers/medication_discharge_importer.rb'
-require_relative 'qrda-import/data-element-importers/medication_dispensed_importer.rb'
-require_relative 'qrda-import/data-element-importers/patient_characteristic_expired.rb'
-require_relative 'qrda-import/data-element-importers/procedure_order_importer.rb'
-require_relative 'qrda-import/data-element-importers/substance_administered_importer.rb'
-require_relative 'qrda-import/patient_importer.rb'
 require_relative 'ext/data_element.rb'
-require_relative 'ext/code.rb'
+
+require_relative 'measure-loader/helpers'
+require_relative 'measure-loader/cql_loader'
+require_relative 'measure-loader/elm_dependency_finder'
+require_relative 'measure-loader/elm_parser'
+require_relative 'measure-loader/exceptions'
+require_relative 'measure-loader/hqmf_measure_loader'
+require_relative 'measure-loader/mat_measure_files'
+require_relative 'measure-loader/source_data_criteria_loader'
+require_relative 'measure-loader/value_set_helpers'
+require_relative 'measure-loader/vsac_value_set_loader'

data/lib/hqmf-parser/2.0/document.rb

@@ -192,7 +192,7 @@ module HQMF2
       value_obj = handle_attribute_value(attribute, value) if attribute.at_xpath('./cda:value', NAMESPACES)
 
       # Handle the cms_id - changed to eCQM in MAT 5.4 (QDM 5.3)
-      @cms_id = "CMS#{value}v#{@hqmf_version_number.to_i}" if (!name.nil?) && ((name.include? 'eMeasure Identifier') || (name.include? 'eCQM Identifier'))
+      @cms_id = "CMS#{value}v#{@hqmf_version_number.to_i}" if name&.start_with?('eMeasure Identifier', 'eCQM Identifier')
 
       HQMF::Attribute.new(id, code, value, nil, name, id_obj, code_obj, value_obj)
     end

data/lib/hqmf-parser/cql/document_helpers/doc_population_helper.rb

@@ -40,10 +40,15 @@ module HQMF2CQL
           # The at_xpath(...).values returns an array of a single element.
           # The match returns an array and since we don't want the double quotes we take the second element
           cql_define_function[:function_name] = entry.at_xpath("*/cda:measureObservationDefinition/cda:value/cda:expression").values.first.match('\\"([A-Za-z0-9 ]+)\\"')[1]
+          cql_define_function[:function_aggregation_type] = entry.at_xpath("*/cda:measureObservationDefinition/cda:methodCode/cda:item").attributes['code'].value
+          cql_define_function[:function_hqmf_oid] = entry.at_xpath("*/cda:measureObservationDefinition/cda:id").attributes['root'].value
           # The criteria_reference_id is the id of the measurePopulationCriteria that should be used for this observation function
           measure_population_id = entry.at_xpath("*/cda:measureObservationDefinition/cda:component/cda:criteriaReference/cda:id").attributes['root'].value
           # Get the name of the parameter to the  observation function within the measurePopulationCriteria section
-          cql_define_function[:parameter] = @doc.at_xpath("cda:QualityMeasureDocument/cda:component/cda:populationCriteriaSection/cda:component/cda:measurePopulationCriteria/cda:id[@root = \"#{measure_population_id}\"]/../cda:precondition/cda:criteriaReference/cda:id").attributes['extension'].value.match('\\"([A-Za-z0-9 ]+)\\"')[1]
+
+          measure_population_name = entry.at_xpath("*/cda:measureObservationDefinition/cda:component/cda:criteriaReference/cda:id").attributes['extension'].value
+          # Get the name of the parameter to the observation function within the relevant population criteria section
+          cql_define_function[:parameter] = @doc.at_xpath("cda:QualityMeasureDocument/cda:component/cda:populationCriteriaSection/cda:component/cda:#{measure_population_name}Criteria/cda:id[@root = \"#{measure_population_id}\"]/../cda:precondition/cda:criteriaReference/cda:id").attributes['extension'].value.match('\\"([A-Za-z0-9 ]+)\\"')[1]
 
           @observations << cql_define_function
         end

data/lib/measure-loader/cql_loader.rb

@@ -0,0 +1,165 @@
+module Measures
+  class CqlLoader
+
+    def initialize(measure_zip, measure_details, value_set_loader = nil)
+      @measure_zip = measure_zip
+      @measure_details = measure_details.deep_symbolize_keys
+      @vs_model_cache = {}
+      @value_set_loader = value_set_loader
+      @value_set_loader.vs_model_cache = @vs_model_cache if @value_set_loader.present?
+    end
+
+    # Returns an array of measures, will contain a single measure if it is a non-composite measure
+    def extract_measures
+      measure_files = MATMeasureFiles.create_from_zip_file(@measure_zip)
+
+      measures = []
+      if measure_files.components.present?
+        measure, component_measures = create_measure_and_components(measure_files)
+        measures.push(*component_measures)
+      else
+        measure = create_measure(measure_files)
+      end
+      measure.package = CQM::MeasurePackage.new(file: BSON::Binary.new(@measure_zip.read))
+      measures << measure
+
+      measures.each { |m| CqlLoader.update_population_set_and_strat_titles(m, @measure_details[:population_titles]) }
+      return measures
+    end
+
+    def self.update_population_set_and_strat_titles(measure, population_titles)
+      # Sample population_titles: [pop set 1 title, pop set 2 title, pop set 1 strat 1 title,
+      #                   pop set 1 strat 2 title, pop set 2 strat 1 title, pop set 2 strat 2 title]
+      # Note RE composite measures: components and composite must have same population sets and strats
+      return if population_titles.nil? || population_titles.empty?
+      title_idx = 0
+      measure.population_sets.each do |population_set|
+        population_set.title = population_titles[title_idx] if population_titles[title_idx].present?
+        title_idx += 1
+        break if title_idx >= population_titles.size
+      end
+
+      return if title_idx >= population_titles.size
+
+      measure.population_sets.flat_map(&:stratifications).each do |strat|
+        strat.title = population_titles[title_idx] if population_titles[title_idx].present?
+        title_idx += 1
+        break if title_idx >= population_titles.size
+      end
+    end
+
+    private
+
+    def create_measure_and_components(measure_files)
+      top_level_library_ids = measure_files.cql_libraries.map { |lib| "#{lib.id}_v#{lib.version}" }
+      add_component_cql_library_files_to_composite_measure_files(measure_files)
+      measure = create_measure(measure_files)
+      component_measures = create_component_measures(measure_files, measure.hqmf_set_id)
+      measure.component_hqmf_set_ids = component_measures.map(&:hqmf_set_id)
+      unset_top_level_flag_on_cql_libraries_imported_from_components(measure, top_level_library_ids)
+
+      return measure, component_measures
+    end
+
+    def create_component_measures(measure_files, composite_measure_hqmf_set_id)
+      component_measures = measure_files.components.map { |comp_files| create_measure(comp_files) }
+      component_measures.each do |component_measure|
+        # Set the components' hqmf_set_id to: <composite_hqmf_set_id>&<component_hqmf_set_id>
+        component_measure.hqmf_set_id = "#{composite_measure_hqmf_set_id}&#{component_measure.hqmf_set_id}"
+        component_measure.component = true
+        component_measure.composite_hqmf_set_id = composite_measure_hqmf_set_id
+      end
+      return component_measures
+    end
+
+    def unset_top_level_flag_on_cql_libraries_imported_from_components(composite_measure, top_level_library_ids)
+      composite_measure.cql_libraries.each do |lib|
+        unless "#{lib.library_name}_v#{lib.library_version}".in? top_level_library_ids
+          lib.is_top_level = false # is_top_level defaults to true
+        end
+      end
+    end
+
+    def add_component_cql_library_files_to_composite_measure_files(measure_files)
+      component_cql_library_files = measure_files.components.flat_map(&:cql_libraries)
+      measure_files.cql_libraries.push(*component_cql_library_files)
+      measure_files.cql_libraries.uniq! { |cl| cl.id + cl.version }
+      return nil
+    end
+
+    # Creates and returns a measure
+    def create_measure(measure_files)
+      hqmf_xml = measure_files.hqmf_xml
+      # update the valueset info in each elm (update version and remove urn:oid)
+      measure_files.cql_libraries.each { |cql_lib_files| modify_elm_valueset_information(cql_lib_files.elm) }
+
+      measure = CQM::Measure.new(HQMFMeasureLoader.extract_fields(hqmf_xml))
+      measure.cql_libraries = create_cql_libraries(measure_files.cql_libraries, measure.main_cql_library)
+      measure.composite = measure_files.components.present?
+      measure.calculation_method = @measure_details[:episode_of_care] ? 'EPISODE_OF_CARE' : 'PATIENT'
+      measure.calculate_sdes = @measure_details[:calculate_sdes]
+
+      hqmf_model = HQMF::Parser::V2CQLParser.new.parse(hqmf_xml) # TODO: move away from using V2CQLParser
+
+      elms = measure.cql_libraries.map(&:elm)
+      elm_valuesets = ValueSetHelpers.unique_list_of_valuesets_referenced_by_elms(elms)
+      verify_hqmf_valuesets_match_elm_valuesets(elm_valuesets, hqmf_model)
+
+      value_sets_from_single_code_references = ValueSetHelpers.make_fake_valuesets_from_single_code_references(elms, @vs_model_cache)
+      measure.source_data_criteria = SourceDataCriteriaLoader.new(hqmf_xml, value_sets_from_single_code_references).extract_data_criteria
+      measure.value_sets = value_sets_from_single_code_references
+      measure.value_sets.concat(@value_set_loader.retrieve_and_modelize_value_sets_from_vsac(elm_valuesets)) if @value_set_loader.present?
+
+      ## this to_json is needed, it doesn't actually produce json, it just makes a hash that is better
+      ## suited for our uses (e.g. source_data_criteria goes from an array to a hash keyed by id)
+      hqmf_model_hash = hqmf_model.to_json.deep_symbolize_keys!
+      HQMFMeasureLoader.add_fields_from_hqmf_model_hash(measure, hqmf_model_hash)
+
+      return measure
+    end
+
+    def create_cql_libraries(cql_library_files, main_cql_lib)
+      cql_statement_dependencies_all_libs = ElmDependencyFinder.find_dependencies(cql_library_files, main_cql_lib)
+
+      cql_libraries = cql_library_files.map do |cql_lib_files|
+        cql_statement_dependencies = cql_statement_dependencies_all_libs[cql_lib_files.id]
+        is_main_cql_lib = cql_lib_files.id == main_cql_lib
+        modelize_cql_library(cql_lib_files, cql_statement_dependencies, is_main_cql_lib)
+      end
+      return cql_libraries
+    end
+
+    def modelize_cql_library(cql_lib_files, cql_statement_dependencies, is_main_cql_lib)
+      return CQM::CQLLibrary.new(
+        library_name: cql_lib_files.id,
+        library_version: cql_lib_files.version,
+        elm: cql_lib_files.elm,
+        cql: cql_lib_files.cql,
+        elm_annotations: ElmParser.parse(cql_lib_files.elm_xml),
+        statement_dependencies: modelize_cql_statement_dependencies(cql_statement_dependencies),
+        is_main_library: is_main_cql_lib
+      )
+    end
+
+    def modelize_cql_statement_dependencies(cql_statment_deps)
+      return cql_statment_deps.map do |name, refs|
+        refs = refs.map { |ref| CQM::StatementReference.new(ref) }
+        CQM::StatementDependency.new(statement_name: name, statement_references: refs)
+      end
+    end
+
+    def modify_elm_valueset_information(elm)
+      ValueSetHelpers.remove_urnoid(elm)
+      ValueSetHelpers.modify_value_set_versions(elm)
+      return nil
+    end
+
+    def verify_hqmf_valuesets_match_elm_valuesets(elm_valuesets, measure_hqmf_model)
+      # Exclude patient birthdate OID (2.16.840.1.113883.3.117.1.7.1.70) and patient expired
+      # OID (2.16.840.1.113883.3.117.1.7.1.309) used by SimpleXML parser for AGE_AT handling
+      # and bad oid protection in missing VS check
+      missing = (measure_hqmf_model.all_code_set_oids - elm_valuesets.map {|v| v[:oid]} - ['2.16.840.1.113883.3.117.1.7.1.70', '2.16.840.1.113883.3.117.1.7.1.309'])
+      raise MeasureLoadingInvalidPackageException.new("The HQMF file references the following valuesets not present in the CQL: #{missing}") unless missing.empty?
+    end
+  end
+end

data/lib/measure-loader/elm_dependency_finder.rb

@@ -0,0 +1,72 @@
+module Measures
+  module ElmDependencyFinder
+    class << self
+
+      def find_dependencies(cql_library_files, main_cql_library_id)
+        elms = cql_library_files.map(&:elm)
+        all_elms_dep_map = Hash[elms.map { |elm| [Helpers.elm_id(elm), make_statement_deps_for_elm(elm)] }]
+        needed_deps_map = Hash[elms.map { |elm| [Helpers.elm_id(elm), {}] }]
+
+        needed_deps_map[main_cql_library_id] = all_elms_dep_map[main_cql_library_id]
+        needed_deps_map[main_cql_library_id].each_value do |stmnts|
+          stmnts.each { |stmnt| deep_add_external_library_deps(stmnt, needed_deps_map, all_elms_dep_map) }
+        end
+        return needed_deps_map
+      end
+
+      private
+
+      def make_library_alias_to_path_hash(elm)
+        lib_alias_to_path = { nil => Helpers.elm_id(elm) } # nil value used for statements without libraryName
+        (elm.dig('library','includes','def') || []).each do |library_hash|
+          lib_alias_to_path[library_hash['localIdentifier']] = library_hash['path']
+        end
+        return lib_alias_to_path
+      end
+
+      def make_statement_deps_for_elm(elm)
+        deps = {}
+        lib_alias_to_path = make_library_alias_to_path_hash(elm)
+        make_statement_deps_for_elm_helper(elm, nil, deps, lib_alias_to_path)
+        deps.each_value(&:uniq!)
+        return deps
+      end
+
+      def make_statement_deps_for_elm_helper(obj, parent_name, deps, lib_alias_to_path)
+        if obj.is_a? Array
+          obj.each { |el| make_statement_deps_for_elm_helper(el, parent_name, deps, lib_alias_to_path) }
+        elsif obj.is_a? Hash
+          if obj['type'].in?(['ExpressionRef', 'FunctionRef']) && parent_name != 'Patient'
+            dep = { library_name: lib_alias_to_path[obj['libraryName']], statement_name: obj['name'] }
+            deps[parent_name] << dep
+          elsif obj.key?('name') && obj.key?('expression')
+            parent_name = obj['name']
+            deps[parent_name] = [] unless deps.key?('parent_name')
+          end
+          obj.each_pair do |k,v|
+            make_statement_deps_for_elm_helper(v, parent_name, deps, lib_alias_to_path) unless k == 'annotation'
+          end
+        end
+      end
+      
+      def deep_add_external_library_deps(statement, needed_deps_map, all_elms_dep_map)
+        statement_library = statement[:library_name]
+        statement_name = statement[:statement_name]
+
+        return unless needed_deps_map.dig(statement_library, statement_name).nil? # return if key already exists
+
+        if all_elms_dep_map[statement_library].nil?
+          raise MeasureLoadingInvalidPackageException.new("Elm library #{statement_library} referenced but not found.")
+        end
+        if all_elms_dep_map[statement_library][statement_name].nil?
+          raise MeasureLoadingException.new("Elm statement '#{statement_name}' referenced but not found in library '#{statement_library}'.")
+        end
+        deps_to_add = all_elms_dep_map[statement_library][statement_name]
+        needed_deps_map.deep_merge!(statement_library => { statement_name => deps_to_add })
+
+        deps_to_add.each { |stmnt| deep_add_external_library_deps(stmnt, needed_deps_map, all_elms_dep_map) }
+      end
+      
+    end
+  end
+end