onc_certification_g10_test_kit 2.0.0.rc1

data/lib/inferno/terminology/fhir_package_manager.rb

@@ -0,0 +1,69 @@
+require 'rubygems/package'
+require 'tempfile'
+require 'zlib'
+require 'json'
+
+require_relative '../exceptions'
+
+module Inferno
+  module Terminology
+    module FHIRPackageManager
+      class << self
+        REGISTRY_SERVER_URL = 'https://packages.fhir.org'.freeze
+        # Get the FHIR Package from the registry.
+        #
+        # e.g. get_package('hl7.fhir.us.core#3.1.1')
+        #
+        # @param [String] package The FHIR Package
+        def get_package(package, destination, desired_types = [])
+          package_url = package
+            .split('#')
+            .prepend(REGISTRY_SERVER_URL)
+            .join('/')
+
+          tar_file_name = "tmp/#{package.split('#').join('-')}.tgz"
+
+          File.open(tar_file_name, 'w') do |output_file|
+            output_file.binmode
+            block = proc do |response|
+              response.read_body do |chunk|
+                output_file.write chunk
+              end
+            end
+            RestClient::Request.execute(method: :get, url: package_url, block_response: block)
+          end
+
+          tar = Gem::Package::TarReader.new(Zlib::GzipReader.open("tmp/#{package.split('#').join('-')}.tgz"))
+
+          path = File.join destination.split('/')
+          FileUtils.mkdir_p(path)
+
+          tar.each do |entry|
+            next if entry.directory?
+
+            next unless entry.full_name.start_with? 'package/'
+
+            file_name = entry.full_name.split('/').last
+            next if desired_types.present? && !file_name.start_with?(*desired_types)
+
+            resource = JSON.parse(entry.read) if file_name.end_with? '.json'
+            next unless resource&.[]('url')
+
+            encoded_name = "#{encode_name(resource['url'])}.json"
+            encoded_file_name = File.join(path, encoded_name)
+            if File.exist?(encoded_file_name) && !resource['url'] == JSON.parse(File.read(encoded_file_name))['url']
+              raise FileExistsException, "#{encoded_name} already exists for #{resource['url']}"
+            end
+
+            File.write(encoded_file_name, resource.to_json)
+          end
+          File.delete(tar_file_name)
+        end
+
+        def encode_name(name)
+          Zlib.crc32(name)
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/loader.rb

@@ -0,0 +1,298 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/all'
+require 'bloomer'
+require 'bloomer/msgpackable'
+require 'fileutils'
+require 'pry'
+
+require_relative '../exceptions'
+require_relative '../ext/bloomer'
+require_relative '../repositiories/validators'
+require_relative '../repositiories/value_sets'
+require_relative 'fhir_package_manager'
+require_relative 'value_set'
+require_relative 'validator'
+
+module Inferno
+  module Terminology
+    class Loader
+      SKIP_SYS = [
+        'http://hl7.org/fhir/ValueSet/message-events', # has 0 codes
+        'http://hl7.org/fhir/ValueSet/care-team-category', # has 0 codes
+        'http://hl7.org/fhir/ValueSet/action-participant-role', # has 0 codes
+        'http://hl7.org/fhir/ValueSet/example-filter', # has fake property acme-plasma
+        'http://hl7.org/fhir/ValueSet/all-distance-units', # UCUM filter "canonical"
+        'http://hl7.org/fhir/ValueSet/all-time-units', # UCUM filter "canonical"
+        'http://hl7.org/fhir/ValueSet/example-intensional', # Unhandled filter parent =
+        'http://hl7.org/fhir/ValueSet/use-context', # ValueSet contains an unknown ValueSet
+        'http://hl7.org/fhir/ValueSet/media-modality', # ValueSet contains an unknown ValueSet
+        'http://hl7.org/fhir/ValueSet/example-hierarchical' # Example valueset with fake codes
+      ].freeze
+
+      @value_sets_repo = Inferno::Repositories::ValueSets.new
+      @validators_repo = Inferno::Repositories::Validators.new
+
+      @missing_validators = nil
+
+      class << self
+        attr_reader :validators_repo, :value_sets_repo
+
+        def load_value_sets_from_directory(directory, include_subdirectories = false) # rubocop:disable Style/OptionalBooleanParameter
+          directory += '/**/' if include_subdirectories
+          value_set_files = Dir["#{directory}/*.json"]
+          value_set_files.each do |vs_file|
+            next unless JSON.parse(File.read(vs_file))['resourceType'] == 'ValueSet'
+
+            add_value_set_from_file(vs_file)
+          end
+        end
+
+        # Creates the valueset validators, based on the passed in parameters and
+        # the value_sets_repo
+        #
+        # @param type [Symbol] the type of validators to create, either :bloom or
+        #   :csv
+        # @param [String] minimum_binding_strength the lowest binding strength for
+        #   which we should build validators
+        # @param [Boolean] include_umls a flag to determine if we should build
+        #   validators that require UMLS
+        # @param [Boolean] delete_existing a flag to determine whether any
+        #   existing validators of `type` should be deleted before the creation
+        #   tasks run. Default to `true`. If `false`, the existing validators will
+        #   be read in and combined with the validators created in this step.
+        def create_validators(
+          type: :bloom,
+          minimum_binding_strength: 'example',
+          include_umls: true,
+          delete_existing: true
+        )
+          strengths = ['example', 'preferred', 'extensible', 'required'].drop_while do |s|
+            s != minimum_binding_strength
+          end
+          umls_code_systems = Set.new(ValueSet::SAB.keys)
+          root_dir = "resources/terminology/validators/#{type}"
+
+          FileUtils.rm_r(root_dir, force: true) if delete_existing
+          FileUtils.mkdir_p(root_dir)
+
+          vs_validators = get_value_sets(strengths).map do |vs_url, vs|
+            next if SKIP_SYS.include? vs_url
+            next if !include_umls && !umls_code_systems.disjoint?(Set.new(vs.included_code_systems))
+
+            Inferno.logger.debug "Processing #{vs_url}"
+            filename = "#{root_dir}/#{(URI(vs.url).host + URI(vs.url).path).gsub(%r{[./]}, '_')}"
+            begin
+              # Save the validator to file, and get the "new" count of number of codes
+              new_count = save_to_file(vs.value_set, filename, type)
+
+              {
+                url: vs_url,
+                file: name_by_type(File.basename(filename), type),
+                count: new_count,
+                type: type.to_s,
+                code_systems: vs.included_code_systems
+              }
+            rescue UnknownCodeSystemException,
+                   FilterOperationException,
+                   UnknownValueSetException,
+                   URI::InvalidURIError => e
+              Inferno.logger.warn "#{e.message} for ValueSet: #{vs_url}"
+              next
+            end
+          end
+          vs_validators.compact!
+
+          code_systems = vs_validators.flat_map { |validator| validator[:code_systems] }.uniq
+          vs = ValueSet.new(@db)
+
+          cs_validators = code_systems.map do |cs_name|
+            next if SKIP_SYS.include? cs_name
+            next if !include_umls && umls_code_systems.include?(cs_name)
+
+            Inferno.logger.debug "Processing #{cs_name}"
+            begin
+              cs = vs.code_system_set(cs_name)
+              filename = "#{root_dir}/#{bloom_file_name(cs_name)}"
+              new_count = save_to_file(cs, filename, type)
+
+              {
+                url: cs_name,
+                file: name_by_type(File.basename(filename), type),
+                count: new_count,
+                type: type.to_s,
+                code_systems: cs_name
+              }
+            rescue UnknownCodeSystemException,
+                   FilterOperationException,
+                   UnknownValueSetException,
+                   URI::InvalidURIError => e
+              Inferno.logger.warn "#{e.message} for CodeSystem #{cs_name}"
+              next
+            end
+          end
+          validators = (vs_validators + cs_validators).compact
+
+          # Write manifest for loading later
+          File.write("#{root_dir}/manifest.yml", validators.to_yaml)
+
+          create_code_system_metadata(cs_validators.map { |validator| validator[:url] }, root_dir)
+        end
+
+        def create_code_system_metadata(system_urls, root_dir)
+          vs = ValueSet.new(@db)
+          metadata_path = "#{root_dir}/metadata.yml"
+          metadata =
+            if File.file? metadata_path
+              YAML.load_file(metadata_path)
+            else
+              {}
+            end
+          system_urls.each do |url|
+            abbreviation = vs.umls_abbreviation(url)
+            next if abbreviation.nil?
+
+            versions = @db.execute("SELECT SVER FROM mrsab WHERE RSAB='#{abbreviation}' AND SABIN='Y'").flatten
+            restriction_level = @db.execute(
+              "SELECT SRL FROM mrsab WHERE RSAB='#{abbreviation}' AND SABIN='Y'"
+            ).flatten.first
+            system_metadata = metadata[url] || vs.code_system_metadata(url).dup || {}
+            system_metadata[:versions] ||= []
+            system_metadata[:versions].concat(versions).uniq!
+            system_metadata[:restriction_level] = restriction_level
+
+            metadata[url] = system_metadata
+          end
+
+          File.write(metadata_path, metadata.to_yaml)
+        end
+
+        # NOTE: resources/value_sets.yml controls which value sets get loaded.
+        # It is currently manually generated from the US Core metadata.
+        def get_value_sets(strengths)
+          expected_vs_urls =
+            YAML.load_file(File.join('resources', 'value_sets.yml'))
+              .select { |vs| strengths.include? vs[:strength] }
+              .map! { |vs| vs[:system] }
+              .compact
+              .uniq
+
+          available_value_sets = value_sets_repo.select_by_url(expected_vs_urls)
+
+          # Throw an error message for each missing valueset
+          # But don't halt the rake task
+          (expected_vs_urls - available_value_sets.keys).each do |missing_vs_url|
+            Inferno.logger.error "Inferno doesn't know about valueset #{missing_vs_url}"
+          end
+          available_value_sets
+        end
+
+        # Chooses which filetype to save the validator as, based on the type variable passed in
+        def save_to_file(codeset, filename, type)
+          case type
+          when :bloom
+            save_bloom_to_file(codeset, name_by_type(filename, type))
+          when :csv
+            save_csv_to_file(codeset, name_by_type(filename, type))
+          else
+            raise 'Unknown Validator Type!'
+          end
+        end
+
+        def name_by_type(filename, type)
+          case type
+          when :bloom
+            "#{filename}.msgpack"
+          when :csv
+            "#{filename}.csv"
+          else
+            raise 'Unknown Validator Type!'
+          end
+        end
+
+        # Saves the valueset bloomfilter to a msgpack file
+        #
+        # @param [String] filename the name of the file
+        def save_bloom_to_file(codings, filename)
+          # If the file already exists, load it in
+          bloom_filter =
+            if File.file? filename
+              Bloomer::Scalable.from_msgpack(File.read(filename))
+            else
+              Bloomer::Scalable.create_with_sufficient_size(codings.length)
+            end
+          codings.each do |coding|
+            bloom_filter.add_without_duplication("#{coding[:system]}|#{coding[:code]}")
+          end
+          bloom_file = File.new(filename, 'wb')
+          bloom_file.write(bloom_filter.to_msgpack) unless bloom_filter.nil?
+
+          bloom_filter.count
+        end
+
+        # Saves the valueset to a csv
+        # @param [String] filename the name of the file
+        def save_csv_to_file(codings, filename)
+          # If the file already exists, add it to the Set
+          csv_set = Set.new
+          if File.file? filename
+            CSV.read(filename).each do |code_array|
+              csv_set.add({ code: code_array[1], system: code_array[0] })
+            end
+          end
+          codings.merge csv_set
+
+          CSV.open(filename, 'wb') do |csv|
+            codings.each do |coding|
+              csv << [coding[:system], coding[:code]]
+            end
+          end
+          codings.length
+        end
+
+        def register_umls_db(database)
+          FileUtils.mkdir_p File.dirname(database)
+          @db = SQLite3::Database.new database
+        end
+
+        def add_value_set_from_file(vs_file)
+          vs = ValueSet.new(@db)
+          vs.read_value_set(vs_file)
+          value_sets_repo.insert(vs)
+          vs
+        end
+
+        def load_validators(directory = 'resources/terminology/validators/bloom')
+          manifest_file = "#{directory}/manifest.yml"
+          return unless File.file? manifest_file
+
+          validator_metadata = YAML.load_file(manifest_file)
+          validator_metadata.each do |metadata|
+            metadata[:bloom_filter] =
+              Bloomer::Scalable.from_msgpack(File.read("#{directory}/#{metadata[:file]}"))
+            validators_repo.insert(Validator.new(metadata))
+          end
+        end
+
+        def bloom_file_name(codesystem)
+          system = codesystem.tr('|', '_')
+          uri = URI(system)
+          return (uri.host + uri.path).gsub(%r{[./]}, '_') if uri.host && uri.port
+
+          system.gsub(/\W/, '_')
+        end
+
+        def missing_validators
+          return @missing_validators if @missing_validators
+
+          required_value_sets =
+            value_sets_repo
+              .select_by_binding_strength(['required', 'extensible', 'preferred'])
+              .map(&:value_set_url)
+          @missing_validators = required_value_sets.compact - validators_repo.all_urls
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/check_built_terminology.rb

@@ -0,0 +1,77 @@
+require 'yaml'
+
+module Inferno
+  module Terminology
+    module Tasks
+      class CheckBuiltTerminology
+        MIME_TYPE_SYSTEMS = [
+          'http://hl7.org/fhir/ValueSet/mimetypes',
+          'urn:ietf:bcp:13'
+        ].freeze
+
+        def run
+          if mismatched_value_sets.blank?
+            Inferno.logger.info 'Terminology built successfully.'
+            return
+          end
+
+          if only_mime_types_mismatch?
+            Inferno.logger.info <<~MIME
+              Terminology built successfully.
+
+              Mime-type based terminology did not match, but this can be a
+              result of using a newer version of the `mime-types-data` gem and
+              does not necessarily reflect a problem with the terminology build.
+              The expected mime-types codes were generated with version
+              `mime-types-data` version `3.2021.0901`.
+            MIME
+          else
+            Inferno.logger.info 'Terminology build results different than expected.'
+          end
+
+          mismatched_value_sets.each do |value_set|
+            Inferno.logger.info mismatched_value_set_message(value_set)
+          end
+        end
+
+        def expected_manifest
+          YAML.load_file(File.join(__dir__, '..', 'expected_manifest.yml'))
+        end
+
+        def new_manifest_path
+          @new_manifest_path ||=
+            File.join(Dir.pwd, 'resources', 'terminology', 'validators', 'bloom', 'manifest.yml')
+        end
+
+        def new_manifest
+          return [] unless File.exist? new_manifest_path
+
+          YAML.load_file(new_manifest_path)
+        end
+
+        def mismatched_value_sets
+          @mismatched_value_sets ||=
+            expected_manifest.reject do |expected_value_set|
+              url = expected_value_set[:url]
+              new_value_set(url) == expected_value_set
+            end
+        end
+
+        def new_value_set(url)
+          new_manifest.find { |value_set| value_set[:url] == url }
+        end
+
+        def only_mime_types_mismatch?
+          mismatched_value_sets.all? { |value_set| MIME_TYPE_SYSTEMS.include? value_set[:url] }
+        end
+
+        def mismatched_value_set_message(expected_value_set)
+          url = expected_value_set[:url]
+          actual_value_set = new_value_set(url)
+
+          "#{url}: Expected codes: #{expected_value_set[:count]} Actual codes: #{actual_value_set&.dig(:count) || 0}"
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/cleanup.rb

@@ -0,0 +1,13 @@
+module Inferno
+  module Terminology
+    module Tasks
+      class Cleanup
+        def run
+          Inferno.logger.info "removing all terminology build files in #{TEMP_DIR}"
+
+          FileUtils.remove_dir File.join(TEMP_DIR)
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/cleanup_precursors.rb

@@ -0,0 +1,23 @@
+module Inferno
+  module Terminology
+    module Tasks
+      class CleanupPrecursors
+        include TempDir
+
+        attr_reader :version
+
+        def initialize(version:)
+          @version = version
+        end
+
+        def run
+          Inferno.logger.info "removing terminology precursor files in #{versioned_temp_dir}"
+          FileUtils.remove_dir(umls_dir, true)
+          FileUtils.remove_dir(umls_subset_dir, true)
+          FileUtils.rm(umls_zip_path, force: true)
+          FileUtils.rm(pipe_files, force: true)
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/count_codes_in_value_set.rb

@@ -0,0 +1,20 @@
+module Inferno
+  module Terminology
+    module Tasks
+      class CountCodesInValueSet
+        attr_reader :value_set_url
+
+        def initialize(vs:) # rubocop:disable Naming/MethodParameterName
+          @value_set_url = vs
+        end
+
+        def run
+          Loader.register_umls_db File.join(TEMP_DIR, 'umls.db')
+          Loader.load_value_sets_from_directory(PACKAGE_DIR, true)
+          vs = Repositories::ValueSets.new.find(value_set_url)
+          Inferno.logger.info vs&.valueset&.count
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/create_value_set_validators.rb

@@ -0,0 +1,34 @@
+require_relative 'temp_dir'
+
+module Inferno
+  module Terminology
+    module Tasks
+      class CreateValueSetValidators
+        include TempDir
+
+        attr_reader :minimum_binding_strength, :version, :delete_existing, :type
+
+        def initialize(minimum_binding_strength:, version:, delete_existing:, type:)
+          @minimum_binding_strength = minimum_binding_strength
+          @version = version
+          @delete_existing = delete_existing != 'false'
+          @type = type.to_sym
+        end
+
+        def run
+          Loader.register_umls_db db_for_version
+          Loader.load_value_sets_from_directory(Inferno::Terminology::PACKAGE_DIR, true)
+          Loader.create_validators(
+            type: type,
+            minimum_binding_strength: minimum_binding_strength,
+            delete_existing: delete_existing
+          )
+        end
+
+        def db_for_version
+          File.join(versioned_temp_dir, 'umls.db')
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/download_fhir_terminology.rb

@@ -0,0 +1,27 @@
+module Inferno
+  module Terminology
+    module Tasks
+      class DownloadFHIRTerminology
+        def run
+          FileUtils.mkdir_p PACKAGE_DIR
+
+          download_fhir_r4
+          download_fhir_expansions
+          download_us_core
+        end
+
+        def download_fhir_r4
+          FHIRPackageManager.get_package('hl7.fhir.r4.core#4.0.1', PACKAGE_DIR, ['ValueSet', 'CodeSystem'])
+        end
+
+        def download_us_core
+          FHIRPackageManager.get_package('hl7.fhir.us.core#3.1.1', PACKAGE_DIR, ['ValueSet', 'CodeSystem'])
+        end
+
+        def download_fhir_expansions
+          FHIRPackageManager.get_package('hl7.fhir.r4.expansions#4.0.1', PACKAGE_DIR, ['ValueSet', 'CodeSystem'])
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/download_umls.rb

@@ -0,0 +1,109 @@
+require 'nokogiri'
+require 'rest-client'
+require_relative 'temp_dir'
+
+module Inferno
+  module Terminology
+    module Tasks
+      class DownloadUMLS
+        include TempDir
+
+        UMLS_FILE_URLS = {
+          '2019' => 'https://download.nlm.nih.gov/umls/kss/2019AB/umls-2019AB-full.zip',
+          '2020' => 'https://download.nlm.nih.gov/umls/kss/2020AB/umls-2020AB-full.zip',
+          '2021' => 'https://download.nlm.nih.gov/umls/kss/2021AA/umls-2021AA-full.zip'
+        }.freeze
+        TICKET_GRANTING_TICKET_URL = 'https://utslogin.nlm.nih.gov/cas/v1/api-key'.freeze
+
+        attr_reader :version, :api_key
+
+        def initialize(version:, apikey:)
+          @version = version
+          @api_key = apikey
+        end
+
+        def run
+          # Adapted from https://documentation.uts.nlm.nih.gov/automating-downloads.html
+
+          FileUtils.mkdir_p(versioned_temp_dir)
+
+          target_file = UMLS_FILE_URLS[version]
+
+          Inferno.logger.info 'Getting Ticket Granting Ticket'
+          ticket_granting_ticket_html = RestClient::Request.execute(
+            method: :post,
+            url: TICKET_GRANTING_TICKET_URL,
+            payload: {
+              apikey: api_key
+            }
+          )
+          ticket_granting_ticket =
+            Nokogiri::HTML(ticket_granting_ticket_html.body).at_css('form').attributes['action'].value
+
+          Inferno.logger.info 'Getting ticket'
+          ticket = RestClient::Request.execute(
+            method: :post,
+            url: ticket_granting_ticket,
+            payload: {
+              service: target_file
+            }
+          ).body
+
+          begin
+            Inferno.logger.info 'Downloading'
+            RestClient::Request.execute(
+              method: :get,
+              url: "#{target_file}?ticket=#{ticket}",
+              max_redirects: 0
+            )
+          rescue RestClient::ExceptionWithResponse => e
+            ticket = RestClient::Request.execute(
+              method: :post,
+              url: ticket_granting_ticket,
+              payload: {
+                service: e.response.headers[:location]
+              }
+            ).body
+            target_location = umls_zip_path
+            follow_redirect(e.response.headers[:location], target_location, ticket, e.response.headers[:set_cookie])
+          end
+          Inferno.logger.info 'Finished Downloading!'
+        end
+
+        def follow_redirect(location, file_location, ticket, cookie = nil)
+          return unless location
+
+          size = 0
+          percent = 0
+          current_percent = 0
+          File.open(file_location, 'w') do |f|
+            f.binmode
+            block = proc do |response|
+              Inferno.logger.info response.header['content-type']
+              if response.header['content-type'] == 'application/zip'
+                total = response.header['content-length'].to_i
+                response.read_body do |chunk|
+                  f.write chunk
+                  size += chunk.size
+                  percent = ((size * 100) / total).round unless total.zero?
+                  if current_percent != percent
+                    current_percent = percent
+                    Inferno.logger.info "#{percent}% complete"
+                  end
+                end
+              else
+                follow_redirect(response.header['location'], file_location, ticket, response.header['set-cookie'])
+              end
+            end
+            RestClient::Request.execute(
+              method: :get,
+              url: "#{location}?ticket=#{ticket}",
+              headers: { cookie: cookie },
+              block_response: block
+            )
+          end
+        end
+      end
+    end
+  end
+end