cqm-parsers 0.2.3 → 3.1.0.0

data/lib/measure-loader/value_set_helpers.rb

@@ -0,0 +1,68 @@
+module Measures
+  module ValueSetHelpers
+    class << self
+
+      # Adjusting value set version data. If version is profile, set the version to nil
+      def modify_value_set_versions(elm)
+        (elm.dig('library','valueSets','def') || []).each do |value_set|
+          # If value set has a version and it starts with 'urn:hl7:profile:' then set to nil
+          if value_set['version']&.include?('urn:hl7:profile:')
+            value_set['profile'] = URI.decode_www_form_component(value_set['version'].split('urn:hl7:profile:').last)
+            value_set['version'] = nil
+          # If value has a version and it starts with 'urn:hl7:version:' then strip that and keep the actual version value.
+          elsif value_set['version']&.include?('urn:hl7:version:')
+            value_set['version'] = URI.decode_www_form_component(value_set['version'].split('urn:hl7:version:').last)
+          end
+        end
+      end
+
+      # Removes 'urn:oid:' from ELM
+      def remove_urnoid(json)
+        Utilities.deep_traverse_hash(json) { |hash, k, v| hash[k] = v.gsub('urn:oid:', '') if v.is_a?(String) }
+      end
+
+      def unique_list_of_valuesets_referenced_by_elms(elms)
+        elm_value_sets = []
+        elms.each do |elm|
+          elm.dig('library','valueSets','def')&.each do |value_set|
+            elm_value_sets << {oid: value_set['id'], version: value_set['version'], profile: value_set['profile']}
+          end
+        end
+        elm_value_sets.uniq!
+        return elm_value_sets
+      end
+
+      # Add single code references by finding the codes from the elm and creating new ValueSet objects
+      # With a generated GUID as a fake oid.
+      def make_fake_valuesets_from_single_code_references(elms, vs_model_cache)
+        value_sets_from_single_code_references = []
+
+        elms.each do |elm|
+          # Loops over all single codes and saves them as fake valuesets.
+          (elm.dig('library','codes','def') || []).each do |code_reference|
+            # look up the referenced code system
+            code_system_def = elm['library']['codeSystems']['def'].find { |code_sys| code_sys['name'] == code_reference['codeSystem']['name'] }
+            # Generate a unique number as our fake "oid" based on parameters that identify the DRC
+            code_hash = "drc-" + Digest::SHA2.hexdigest("#{code_system_def['id']}#{code_system_def['version']}#{code_reference['id']}")
+
+            cache_key = [code_hash, '']
+            if vs_model_cache[cache_key].nil?
+              concept = CQM::Concept.new(code: code_reference['id'],
+                                         code_system_name: code_system_def['name'],
+                                         code_system_version: code_system_def['version'],
+                                         code_system_oid: code_system_def['id'],
+                                         display_name: code_reference['name'])
+              vs_model_cache[cache_key] = CQM::ValueSet.new(oid: code_hash,
+                                                            display_name: code_reference['name'],
+                                                            version: '',
+                                                            concepts: [concept])
+            end
+            value_sets_from_single_code_references << vs_model_cache[cache_key]
+          end
+        end
+        return value_sets_from_single_code_references
+      end
+
+    end
+  end
+end

data/lib/measure-loader/vsac_value_set_loader.rb

@@ -0,0 +1,97 @@
+module Measures
+  # Utility class for loading value sets
+  class VSACValueSetLoader
+    attr_accessor :vsac_options, :vs_model_cache
+
+    def initialize(options)
+      options.symbolize_keys!
+      @vsac_options = options[:options]
+      @vsac_ticket_granting_ticket = options[:ticket_granting_ticket]
+      @vsac_username = options[:username]
+      @vsac_password = options[:password]
+      @vs_model_cache = {}
+    end
+
+    def retrieve_and_modelize_value_sets_from_vsac(value_sets)
+      vs_models = []
+      needed_value_sets = []
+
+      value_sets.each do |value_set|
+        vs_vsac_options = make_specific_value_set_options(value_set)
+        query_version = determine_query_version(vs_vsac_options)
+
+        cache_key = [value_set[:oid], query_version]
+        vs_model = @vs_model_cache[cache_key]
+        if vs_model.present?
+          vs_models << vs_model
+        else
+          needed_value_sets << {value_set:  value_set,
+                                vs_vsac_options: vs_vsac_options,
+                                query_version: query_version,
+                                cache_key: cache_key}
+        end
+      end
+
+      vs_responses = load_api.get_multiple_valuesets(needed_value_sets)
+
+      [needed_value_sets,vs_responses].transpose.each do |needed_vs,vs_data|
+        vs_model = modelize_value_set(vs_data, needed_vs[:query_version])
+        @vs_model_cache[needed_vs[:cache_key]] = vs_model
+        vs_models << vs_model
+      end
+
+      puts "\tloaded #{needed_value_sets.size} value sets from vsac, #{value_sets.size - needed_value_sets.size} from cache"
+      return vs_models
+    end
+
+    private
+
+    def load_api
+      return @api if @api.present?
+      @api = Util::VSAC::VSACAPI.new(config: APP_CONFIG['vsac'], ticket_granting_ticket: @vsac_ticket_granting_ticket, username: @vsac_username, password: @vsac_password)
+      return @api
+    end
+
+    def determine_query_version(vs_vsac_options)
+      return "Draft" if vs_vsac_options[:include_draft] == true
+      return "Profile:#{vs_vsac_options[:profile]}" if vs_vsac_options[:profile]
+      return vs_vsac_options[:version] if vs_vsac_options[:version]
+      return "Release:#{vs_vsac_options[:release]}" if vs_vsac_options[:release]
+      return ""
+    end
+
+    def make_specific_value_set_options(value_set)
+      # If we are allowing measure_defined value sets, determine vsac_options for this value set based on elm info.
+      if @vsac_options[:measure_defined]
+        return { profile: value_set[:profile] } unless value_set[:profile].nil?
+        return { version: value_set[:version] } unless value_set[:version].nil?
+      end
+      return @vsac_options
+    end
+
+    def modelize_value_set(vsac_xml_response, query_version)
+      doc = Nokogiri::XML(vsac_xml_response)
+      doc.root.add_namespace_definition("vs","urn:ihe:iti:svs:2008")
+      vs_element = doc.at_xpath("/vs:RetrieveValueSetResponse/vs:ValueSet|/vs:RetrieveMultipleValueSetsResponse/vs:DescribedValueSet")
+      vs = CQM::ValueSet.new(
+        oid: vs_element["ID"],
+        display_name: vs_element["displayName"],
+        version: vs_element["version"] == "N/A" ? query_version : vs_element["version"],
+        concepts: extract_concepts(vs_element)
+      )
+      return vs
+    end
+
+    def extract_concepts(vs_element)
+      concepts = vs_element.xpath("//vs:Concept").collect do |con|
+        CQM::Concept.new(code: con["code"],
+                         code_system_name: con["codeSystemName"],
+                         code_system_version: con["codeSystemVersion"],
+                         code_system_oid: con["codeSystem"],
+                         display_name: con["displayName"])
+      end
+      return concepts
+    end
+
+  end
+end

data/lib/tasks/hqmf.rake

@@ -1,7 +1,7 @@
 require 'pathname'
 require 'fileutils'
 require 'json'
-require 'hqmf-parser'
+require 'cqm-parsers'
 
 namespace :hqmf do
 

data/lib/util/util.rb

@@ -0,0 +1,23 @@
+module Utilities
+  # Traverse each key, value of the hash and any nested hashes (including those in arrays)
+  # E.G. to deep transfrom do:
+  #   deep_traverse_hash(obj) { |hash, k, v| hash[k] = v.upcase if v.is_a?(String) }
+  def self.deep_traverse_hash(obj, &block)
+    if obj.is_a? Array
+      obj.each { |val| deep_traverse_hash(val, &block) }
+    elsif obj.is_a?(Hash)
+      obj.each_pair do |k,v|
+        deep_traverse_hash(v, &block)
+        block.call(obj, k, v)
+      end
+    end
+  end
+
+  def self.remove_enclosing_quotes(str)
+    quote_chars = ['"', '\'']
+    quote_chars.each do |quote_char|
+      return str[1..-2] if str.start_with?(quote_char) && str.end_with?(quote_char)
+    end
+    return str
+  end 
+end

data/lib/util/vsac_api.rb

@@ -1,4 +1,4 @@
-require 'rest_client'
+require 'typhoeus'
 require 'uri'
 
 module Util
@@ -16,6 +16,13 @@ module Util
       end
     end
 
+    # When VSAC responds with a 404
+    class VSACNotFoundError < VSACError
+      def initialize
+        super('Resource not found.')
+      end
+    end
+
     # Error represnting a program not found response from the API.
     class VSACProgramNotFoundError < VSACError
       attr_reader :oid
@@ -80,11 +87,10 @@ module Util
       def initialize(options)
         # check that :config exists and has needed fields
         raise VSACArgumentError.new("Required param :config is missing or empty.") if options[:config].nil?
-        symbolized_config = options[:config].symbolize_keys
-        unless check_config symbolized_config
+        @config = options[:config].symbolize_keys
+        unless check_config @config
           raise VSACArgumentError.new("Required param :config is missing required URLs.")
         end
-        @config = symbolized_config
 
         # if a ticket_granting_ticket was passed in, check it and raise errors if found
         # username and password will be ignored
@@ -94,12 +100,8 @@ module Util
             raise VSACArgumentError.new("Optional param :ticket_granting_ticket is missing :ticket or :expires")
           end
 
-          # check if it has expired
-          if Time.now > provided_ticket_granting_ticket[:expires]
-            raise VSACTicketExpiredError.new
-          end
+          raise VSACTicketExpiredError.new if Time.now > provided_ticket_granting_ticket[:expires]
 
-          # ticket granting ticket looks good
           @ticket_granting_ticket = { ticket: provided_ticket_granting_ticket[:ticket],
                                       expires: provided_ticket_granting_ticket[:expires] }
 
@@ -114,17 +116,12 @@ module Util
       #
       # Returns a list of profile names. These are kept in the order that VSAC provides them in.
       def get_profile_names
-        profiles_response = RestClient.get("#{@config[:utility_url]}/profiles")
-        profiles = []
+        profiles_response = http_get("#{@config[:utility_url]}/profiles")
 
         # parse xml response and get text content of each profile element
         doc = Nokogiri::XML(profiles_response)
         profile_list = doc.at_xpath("/ProfileList")
-        profile_list.xpath("//profile").each do |profile|
-          profiles << profile.text
-        end
-
-        return profiles
+        return profile_list.xpath("//profile").map(&:text)
       end
 
       ##
@@ -132,38 +129,25 @@ module Util
       #
       # Returns a list of program names. These are kept in the order that VSAC provides them in.
       def get_program_names
-        programs_response = RestClient.get("#{@config[:utility_url]}/programs")
-        program_names = []
-
-        # parse json response and return the names of the programs
-        programs_info = JSON.parse(programs_response)['Program']
-        programs_info.each do |program|
-          program_names << program['name']
-        end
-
-        return program_names
+        programs_response = http_get_json("#{@config[:utility_url]}/programs")
+        programs_info = programs_response['Program']
+        return programs_info.map { |program| program['name'] }
       end
 
       ##
       # Gets the details for a program. This may be used without credentials.
       #
       # Optional parameter program is the program to request from the API. If it is not provided it will look for
-      # a :program in the config passed in during construction. If there is no :program in the config it will use 
+      # a :program in the config passed in during construction. If there is no :program in the config it will use
       # the DEFAULT_PROGRAM constant for the program.
       #
       # Returns the JSON parsed response for program details.
       def get_program_details(program = nil)
         # if no program was provided use the one in the config or default in constant
-        if program.nil?
-          program = @config.fetch(:program, DEFAULT_PROGRAM)
-        end
-
-        begin
-          # parse json response and return it
-          return JSON.parse(RestClient.get("#{@config[:utility_url]}/program/#{ERB::Util.url_encode(program)}"))
-        rescue RestClient::ResourceNotFound
-          raise VSACProgramNotFoundError.new(program)
-        end
+        program = @config.fetch(:program, DEFAULT_PROGRAM) if program.nil?
+        return http_get_json("#{@config[:utility_url]}/program/#{ERB::Util.url_encode(program)}")
+      rescue VSACNotFoundError
+        raise VSACProgramNotFoundError.new(program)
       end
 
       ##
@@ -175,29 +159,21 @@ module Util
       #   }
       #
       # Optional parameter program is the program to request from the API. If it is not provided it will look for
-      # a :program in the config passed in during construction. If there is no :program in the config it will use 
+      # a :program in the config passed in during construction. If there is no :program in the config it will use
       # the DEFAULT_PROGRAM constant for the program.
       #
       # Returns the name of the latest profile for the given program.
       def get_latest_profile_for_program(program = nil)
         # if no program was provided use the one in the config or default in constant
-        if program.nil?
-          program = @config.fetch(:program, DEFAULT_PROGRAM)
-        end
+        program = @config.fetch(:program, DEFAULT_PROGRAM) if program.nil?
 
-        begin
-          # parse json response and return it
-          parsed_response = JSON.parse(RestClient.get("#{@config[:utility_url]}/program/#{ERB::Util.url_encode(program)}/latest%20profile"))
+        # parse json response and return it
+        parsed_response = http_get_json("#{@config[:utility_url]}/program/#{ERB::Util.url_encode(program)}/latest%20profile")
 
-          # As of 5/17/18 VSAC does not return 404 when an invalid profile is provided. It just doesnt fill the name
-          # attribute in the 200 response. We need to check this.
-          raise VSACProgramNotFoundError.new(program) if parsed_response['name'].nil?
-          return parsed_response['name']
-
-        # keeping this rescue block in case the API is changed to return 404 for invalid profile
-        rescue RestClient::ResourceNotFound
-          raise VSACProgramNotFoundError.new(program)
-        end
+        # As of 5/17/18 VSAC does not return 404 when an invalid profile is provided. It just doesnt fill the name
+        # attribute in the 200 response. We need to check this.
+        raise VSACProgramNotFoundError.new(program) if parsed_response['name'].nil?
+        return parsed_response['name']
       end
 
       ##
@@ -210,81 +186,168 @@ module Util
       # Returns a list of release names in a program. These are kept in the order that VSAC provides them in.
       def get_program_release_names(program = nil)
         program_details = get_program_details(program)
-        releases = []
-
-        # pull just the release names out
-        program_details['release'].each do |release|
-          releases << release['name']
-        end
-
-        return releases
+        return program_details['release'].map { |release| release['name'] }
       end
 
       ##
       # Gets a valueset. This requires credentials.
       #
       def get_valueset(oid, options = {})
-        # base parameter oid is always needed
-        params = { id: oid }
+        needed_vs = {value_set: {oid: oid}, vs_vsac_options: options}
+        return get_multiple_valuesets([needed_vs])[0]
+      end
 
-        # release parameter, should be used moving forward
-        unless options[:release].nil?
-          params[:release] = options[:release]
-        end
+      ##
+      # Get multiple valuesets (executed in parallel). Requires credentials.
+      #
+      # Parameter needed_value_sets is an array of hashes, each hash should have at least:
+      # hash = {vs_vsac_options: ___, value_set: {oid: ___} }
+      #
+      def get_multiple_valuesets(needed_value_sets)
+        raise VSACNoCredentialsError.new unless @ticket_granting_ticket
+        raise VSACTicketExpiredError.new if Time.now > @ticket_granting_ticket[:expires]
 
-        # profile parameter, may be needed for getting draft value sets
-        if !options[:profile].nil?
-          params[:profile] = options[:profile]
-          unless options[:include_draft].nil?
-            params[:includeDraft] = !options[:include_draft].nil? ? 'yes' : 'no'
-          end
-        else
-          unless options[:include_draft].nil?
-            raise VSACArgumentError.new("Option :include_draft requires :profile to be provided.")
-          end
+        vs_responses = get_multiple_valueset_raw_responses(needed_value_sets)
+        vs_datas = [needed_value_sets,vs_responses].transpose.map do |needed_vs,vs_response|
+          expected_oid = needed_vs[:value_set][:oid]
+          process_and_validate_vsac_response(vs_response, expected_oid)
         end
 
-        # version parameter, rarely used
-        unless options[:version].nil?
-          params[:version] = options[:version]
-        end
+        return vs_datas
+      end
+
+      private
 
-        # get a new service ticket
-        params[:ticket] = get_ticket
+      # Given a raw valueset response, process and validate
+      def process_and_validate_vsac_response(vs_response, expected_oid)
+        raise VSNotFoundError.new(expected_oid) if vs_response.response_code == 404
+        validate_http_status_for_ticket_based_request(vs_response.response_code)
 
-        # run request
+        vs_data = vs_response.body.force_encoding("utf-8")
         begin
-          return RestClient.get("#{@config[:content_url]}/RetrieveMultipleValueSets", params: params)
-        rescue RestClient::ResourceNotFound
-          raise VSNotFoundError.new(oid)
-        rescue RestClient::InternalServerError
-          raise VSACError.new("Server error response from VSAC for (#{oid}).")
+          doc = Nokogiri::XML(vs_data)
+          doc.root.add_namespace_definition("vs","urn:ihe:iti:svs:2008")
+          vs_element = doc.at_xpath("/vs:RetrieveValueSetResponse/vs:ValueSet|/vs:RetrieveMultipleValueSetsResponse/vs:DescribedValueSet")
+          concepts = vs_element.xpath("//vs:Concept")
+        rescue StandardError
+          raise VSACError.new("Could not parse VSAC response for #{expected_oid}. Body: #{vs_data}")
         end
+
+        raise Util::VSAC::VSNotFoundError.new(expected_oid) unless (vs_element && vs_element['ID'] == expected_oid)
+        raise Util::VSAC::VSEmptyError.new(expected_oid) if concepts.empty?
+        return vs_data
       end
 
-      private
+      # Execute bulk requests for valuesets, return the raw Typheous responses (requests executed in parallel)
+      def get_multiple_valueset_raw_responses(needed_value_sets)
+        service_tickets = get_service_tickets(needed_value_sets.size)
 
-      def get_ticket
-        # if there is no ticket granting ticket then we should raise an error
+        hydra = Typhoeus::Hydra.new(max_concurrency: 1) # Hydra executes multiple HTTP requests at once
+        requests = needed_value_sets.map do |n|
+          request = create_valueset_request(n[:value_set][:oid], service_tickets.pop, n[:vs_vsac_options])
+          hydra.queue(request)
+          request
+        end
+
+        hydra.run
+        responses = requests.map(&:response)
+        return responses
+      end
+
+      # Bulk get an amount of service tickets (requests executed in parallel)
+      def get_service_tickets(amount)
         raise VSACNoCredentialsError.new unless @ticket_granting_ticket
-        # if the ticket granting ticket has expired, throw an error
         raise VSACTicketExpiredError.new if Time.now > @ticket_granting_ticket[:expires]
 
-        # attempt to get a ticket
-        begin
-          ticket = RestClient.post("#{@config[:auth_url]}/Ticket/#{@ticket_granting_ticket[:ticket]}", service: TICKET_SERVICE_PARAM)
-          return ticket.to_s
-        rescue RestClient::Unauthorized
+        hydra = Typhoeus::Hydra.new # Hydra executes multiple HTTP requests at once
+        requests = amount.times.map do
+          request = create_service_ticket_request
+          hydra.queue(request)
+          request
+        end
+
+        hydra.run
+        tickets = requests.map do |request|
+          validate_http_status_for_ticket_based_request(request.response.response_code)
+          request.response.body
+        end
+        return tickets
+      end
+
+      # Create a typheous request for a valueset (this must be executed later)
+      def create_valueset_request(oid, ticket, options = {})
+        # base parameter oid is always needed
+        params = { id: oid }
+        # release parameter, should be used moving forward
+        params[:release] = options[:release] unless options[:release].nil?
+
+        # profile parameter, may be needed for getting draft value sets
+        if options[:profile].present?
+          params[:profile] = options[:profile]
+          params[:includeDraft] = options[:include_draft] ? 'yes' : 'no' unless options[:include_draft].nil?
+        end
+        if !options[:include_draft].nil? && options[:profile].nil?
+          raise VSACArgumentError.new("Option :include_draft requires :profile to be provided.")
+        end
+
+        # version parameter, rarely used
+        params[:version] = options[:version] unless options[:version].nil?
+        params[:ticket] = ticket
+
+        return Typhoeus::Request.new("#{@config[:content_url]}/RetrieveMultipleValueSets", params: params)
+      end
+
+      # Create a typheous request for a service ticket (this must be executed later)
+      def create_service_ticket_request
+        return Typhoeus::Request.new("#{@config[:auth_url]}/Ticket/#{@ticket_granting_ticket[:ticket]}",
+                                     method: :post,
+                                     params: { service: TICKET_SERVICE_PARAM})
+      end
+
+      # Use your username and password to retrive a ticket granting ticket from VSAC
+      def get_ticket_granting_ticket(username, password)
+        response = Typhoeus.post(
+          "#{@config[:auth_url]}/Ticket",
+          # looks like typheous sometimes switches the order of username/password when encoding
+          # which vsac cant handle (!?), so encode first
+          body: URI.encode_www_form(username: username, password: password)
+        )
+        raise VSACInvalidCredentialsError.new if response.response_code == 401
+        validate_http_status(response.response_code)
+        return { ticket: String.new(response.body), expires: Time.now + 8.hours }
+      end
+
+      # Raise errors if http_status is not OK (200), and expire TGT if auth fails
+      def validate_http_status_for_ticket_based_request(http_status)
+        if http_status == 401
           @ticket_granting_ticket[:expires] = Time.now
           raise VSACTicketExpiredError.new
         end
+        validate_http_status(http_status)
       end
 
-      def get_ticket_granting_ticket(username, password)
-        ticket = RestClient.post("#{@config[:auth_url]}/Ticket", username: username, password: password)
-        return { ticket: String.new(ticket), expires: Time.now + 8.hours }
-      rescue RestClient::Unauthorized
-        raise VSACInvalidCredentialsError.new
+      # Raise errors if http_status is not OK (200)
+      def validate_http_status(http_status)
+        return if http_status == 200
+        if http_status == 0
+          raise VSACError.new("Error communicating with VSAC.")
+        elsif http_status == 404
+          raise VSACNotFoundError.new
+        else
+          raise VSACError.new("HTTP Error code #{http_status} received from VSAC.")
+        end
+      end
+
+      # Convenience function to perform an http get request (raises errors on failure)
+      def http_get(url)
+        response = Typhoeus.get(url)
+        validate_http_status(response.response_code)
+        return response.body
+      end
+
+      # Convenience function to perform an http get request and convert to JSON (raises errors on failure)
+      def http_get_json(url)
+        return JSON.parse(http_get(url))
       end
 
       # Checks to ensure the API config has all necessary fields