onc_certification_g10_test_kit 2.0.0.rc1

data/lib/inferno/terminology/tasks/download_umls_notice.rb

@@ -0,0 +1,20 @@
+module Inferno
+  module Terminology
+    module Tasks
+      module DownloadUMLSNotice
+        def download_umls_notice
+          Inferno.logger.info <<~NOTICE
+            UMLS file not found. Download the US National Library of Medicine (NLM) Unified
+            Medical Language System (UMLS) Full Release files:
+            https://www.nlm.nih.gov/research/umls/licensedcontent/umlsknowledgesources.html
+
+            Install the metathesaurus with the following data sources:
+            CVX|CVX;ICD10CM|ICD10CM;ICD10PCS|ICD10PCS;ICD9CM|ICD9CM;LNC|LNC;MTHICD9|ICD9CM;RXNORM|RXNORM;SNOMEDCT_US|SNOMEDCT;CPT;HCPCS
+            After installation, copy `{install path}/META/MRCONSO.RRF` into your
+            `./tmp/terminology` folder, and rerun this task.
+          NOTICE
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/expand_value_set_to_file.rb

@@ -0,0 +1,36 @@
+module Inferno
+  module Terminology
+    module Tasks
+      class ExpandValueSetToFile
+        attr_reader :filename, :type, :value_set_url
+
+        def initialize(vs:, filename:, type:) # rubocop:disable Naming/MethodParameterName
+          @value_set_url = vs
+          @filename = filename
+          @type = type
+        end
+
+        def run
+          # JSON is a special case, because we need to add codes from valuesets from several versions
+          # We accomplish this by collecting and merging codes from each version
+          # Before writing the JSON to a file at the end
+          end_vs = nil if type == 'json'
+
+          %w[2019 2020 2021].each do |version|
+            Loader.register_umls_db File.join(TEMP_DIR, version, 'umls.db')
+            Loader.load_value_sets_from_directory(PACKAGE_DIR, true)
+            vs = Repositories::ValueSets.new.find(value_set_url)
+            if type == 'json'
+              end_vs ||= vs
+              end_vs.value_set.merge vs.value_set
+            else
+              Loader.save_to_file(vs.valueset, filename, type.to_sym)
+            end
+          end
+
+          File.open("#{filename}.json", 'wb') { |f| f << end_vs.expansion_as_fhir_valueset.to_json } if type == 'json'
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/process_umls.rb

@@ -0,0 +1,91 @@
+require 'csv'
+require 'find'
+require_relative 'download_umls_notice'
+require_relative 'temp_dir'
+
+module Inferno
+  module Terminology
+    module Tasks
+      class ProcessUMLS
+        include TempDir
+        include DownloadUMLSNotice
+
+        attr_reader :version
+
+        def initialize(version:)
+          @version = version
+        end
+
+        def run # rubocop:disable Metrics/CyclomaticComplexity
+          Inferno.logger.info 'Looking for `./tmp/terminology/MRCONSO.RRF`...'
+          input_file = Find.find(versioned_temp_dir).find { |f| /MRCONSO.RRF$/ =~f }
+          if input_file
+            start = Time.now
+            output_filename = File.join(versioned_temp_dir, 'terminology_umls.txt')
+            output = File.open(output_filename, 'w:UTF-8')
+            line = 0
+            excluded = 0
+            excluded_systems = Hash.new(0)
+            begin
+              Inferno.logger.info "Writing to #{output_filename}..."
+              CSV.foreach(input_file, headers: false, col_sep: '|', quote_char: "\x00") do |row|
+                line += 1
+                include_code = false
+                code_system = row[11]
+                code = row[13]
+                description = row[14]
+                case code_system
+                when 'SNOMEDCT_US'
+                  code_system = 'SNOMED'
+                  include_code = (row[4] == 'PF' && ['FN', 'OAF'].include?(row[12]))
+                when 'LNC'
+                  code_system = 'LOINC'
+                  include_code = true
+                when 'ICD10CM', 'ICD10PCS'
+                  code_system = 'ICD10'
+                  include_code = (row[12] == 'PT')
+                when 'ICD9CM'
+                  code_system = 'ICD9'
+                  include_code = (row[12] == 'PT')
+                when 'CPT', 'HCPCS'
+                  include_code = (row[12] == 'PT')
+                when 'MTHICD9'
+                  code_system = 'ICD9'
+                  include_code = true
+                when 'RXNORM'
+                  include_code = true
+                when 'CVX'
+                  include_code = ['PT', 'OP'].include?(row[12])
+                when 'SRC'
+                  # 'SRC' rows define the data sources in the file
+                  include_code = false
+                else
+                  include_code = false
+                  excluded_systems[code_system] += 1
+                end
+                if include_code
+                  output.write("#{code_system}|#{code}|#{description}\n")
+                else
+                  excluded += 1
+                end
+              end
+            rescue StandardError => e
+              Inferno.logger.info "Error at line #{line}"
+              Inferno.logger.info e.message
+            end
+            output.close
+            Inferno.logger.info "Processed #{line} lines, excluding #{excluded} redundant entries."
+            Inferno.logger.info "Excluded code systems: #{excluded_systems}" unless excluded_systems.empty?
+            finish = Time.now
+            minutes = ((finish - start) / 60)
+            seconds = (minutes - minutes.floor) * 60
+            Inferno.logger.info "Completed in #{minutes.floor} minute(s) #{seconds.floor} second(s)."
+            Inferno.logger.info 'Done.'
+          else
+            download_umls_notice
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/process_umls_translations.rb

@@ -0,0 +1,85 @@
+require 'find'
+require_relative 'download_umls_notice'
+require_relative 'temp_dir'
+
+module Inferno
+  module Terminology
+    module Tasks
+      class ProcessUMLSTranslations
+        include DownloadUMLSNotice
+
+        def run # rubocop:disable Metrics/CyclomaticComplexity
+          Inferno.logger.info 'Looking for `./tmp/terminology/MRCONSO.RRF`...'
+          input_file = Find.find(File.join(TEMP_DIR, 'terminology')).find { |f| /MRCONSO.RRF$/ =~f }
+          if input_file
+            start = Time.now
+            output_filename = File.join(TEMP_DIR, 'translations_umls.txt')
+            output = File.open(output_filename, 'w:UTF-8')
+            line = 0
+            excluded_systems = Hash.new(0)
+            begin
+              entire_file = File.read(input_file)
+              Inferno.logger.info "Writing to #{output_filename}..."
+              current_umls_concept = nil
+              translation = Array.new(10)
+              entire_file.split("\n").each do |l|
+                row = l.split('|')
+                line += 1
+                concept = row[0]
+                if concept != current_umls_concept && !current_umls_concept.nil?
+                  output.write("#{translation.join('|')}\n") unless translation[1..-2].compact.length < 2
+                  translation = Array.new(10)
+                  current_umls_concept = concept
+                  translation[0] = current_umls_concept
+                elsif current_umls_concept.nil?
+                  current_umls_concept = concept
+                  translation[0] = current_umls_concept
+                end
+                code_system = row[11]
+                code = row[13]
+                translation[9] = row[14]
+                case code_system
+                when 'SNOMEDCT_US'
+                  translation[1] = code if row[4] == 'PF' && ['FN', 'OAF'].include?(row[12])
+                when 'LNC'
+                  translation[2] = code
+                when 'ICD10CM', 'ICD10PCS'
+                  translation[3] = code if row[12] == 'PT'
+                when 'ICD9CM'
+                  translation[4] = code if row[12] == 'PT'
+                when 'MTHICD9'
+                  translation[4] = code
+                when 'RXNORM'
+                  translation[5] = code
+                when 'CVX'
+                  translation[6] = code if ['PT', 'OP'].include?(row[12])
+                when 'CPT'
+                  translation[7] = code if row[12] == 'PT'
+                when 'HCPCS'
+                  translation[8] = code if row[12] == 'PT'
+                when 'SRC'
+                  # 'SRC' rows define the data sources in the file
+                else
+                  excluded_systems[code_system] += 1
+                end
+              end
+            rescue StandardError => e
+              Inferno.logger.info "Error at line #{line}"
+              Inferno.logger.info e.message
+            end
+            output.close
+            Inferno.logger.info "Processed #{line} lines."
+            Inferno.logger.info "Excluded code systems: #{excluded_systems}" unless excluded_systems.empty?
+            finish = Time.now
+            minutes = ((finish - start) / 60)
+            seconds = (minutes - minutes.floor) * 60
+            Inferno.logger.info "Completed in #{minutes.floor} minute(s) #{seconds.floor} second(s)."
+            Inferno.logger.info 'Done.'
+          else
+            download_umls_notice
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/run_umls_jar.rb

@@ -0,0 +1,75 @@
+require_relative 'temp_dir'
+
+module Inferno
+  module Terminology
+    module Tasks
+      # More information on batch running UMLS
+      # https://www.nlm.nih.gov/research/umls/implementation_resources/community/mmsys/BatchMetaMorphoSys.html
+      class RunUMLSJar
+        include TempDir
+
+        VERSIONED_PROPS = {
+          '2019' => 'inferno_2019.prop',
+          '2020' => 'inferno_2020.prop',
+          '2021' => 'inferno_2021.prop'
+        }.freeze
+
+        attr_reader :version
+
+        def initialize(version:)
+          @version = version
+        end
+
+        def run
+          Inferno.logger.info "#{platform} system detected"
+          config_file = File.join(Dir.pwd, 'resources', VERSIONED_PROPS[version])
+          output_dir = File.join(Dir.pwd, versioned_temp_dir, 'umls_subset')
+          FileUtils.mkdir(output_dir)
+
+          Inferno.logger.info "Using #{config_file}"
+          Dir.chdir(Dir[File.join(Dir.pwd, versioned_temp_dir, '/umls/20*')][0]) do
+            Inferno.logger.info Dir.pwd
+            Dir['lib/*.jar'].each do |jar|
+              File.chmod(0o555, jar)
+            end
+            Dir["jre/#{platform}/bin/*"].each do |file|
+              File.chmod(0o555, file)
+            end
+
+            Inferno.logger.info 'Running MetamorphoSys (this may take a while)...'
+            output = system("./jre/#{platform}/bin/java " \
+                            '-Djava.awt.headless=true ' \
+                            '-cp .:lib/jpf-boot.jar ' \
+                            '-Djpf.boot.config=./etc/subset.boot.properties ' \
+                            '-Dlog4j.configuration=./etc/log4j.properties ' \
+                            '-Dinput.uri=. ' \
+                            "-Doutput.uri=#{output_dir} " \
+                            "-Dmmsys.config.uri=#{config_file} " \
+                            '-Xms300M -Xmx8G ' \
+                            'org.java.plugin.boot.Boot')
+            unless output
+              Inferno.logger.info 'MetamorphoSys run failed'
+              # The cwd at this point is 2 directories above where umls_subset
+              # is, so we have to navigate up to it
+              umls_subset_directory = File.join(Dir.pwd, '..', '..', 'umls_subset')
+              FileUtils.remove_dir(umls_subset_directory) if File.directory?(umls_subset_directory)
+              exit 1
+            end
+          end
+
+          Inferno.logger.info 'done'
+        end
+
+        def platform
+          if !(/cygwin|mswin|mingw|bccwin|wince|emx/ =~ RUBY_PLATFORM).nil?
+            'windows64'
+          elsif !(/darwin/ =~ RUBY_PLATFORM).nil?
+            'macos'
+          else
+            'linux'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/temp_dir.rb

@@ -0,0 +1,27 @@
+module Inferno
+  module Terminology
+    module Tasks
+      module TempDir
+        def versioned_temp_dir
+          File.join(TEMP_DIR, version)
+        end
+
+        def umls_zip_path
+          File.join(versioned_temp_dir, 'umls.zip')
+        end
+
+        def umls_dir
+          File.join(versioned_temp_dir, 'umls')
+        end
+
+        def umls_subset_dir
+          File.join(versioned_temp_dir, 'umls_subset')
+        end
+
+        def pipe_files
+          Dir.glob(File.join(versioned_temp_dir, '*.pipe'))
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/unzip_umls.rb

@@ -0,0 +1,42 @@
+require 'zip'
+require_relative 'temp_dir'
+
+module Inferno
+  module Terminology
+    module Tasks
+      class UnzipUMLS
+        include TempDir
+
+        attr_reader :version
+
+        def initialize(version:)
+          @version = version
+        end
+
+        def run
+          # https://stackoverflow.com/questions/19754883/how-to-unzip-a-zip-file-containing-folders-and-files-in-rails-while-keeping-the
+          Zip::File.open(umls_zip_path) do |zip_file|
+            # Handle entries one by one
+            zip_file.each do |entry|
+              # Extract to file/directory/symlink
+              Inferno.logger.info "Extracting #{entry.name}"
+              f_path = File.join(umls_dir, entry.name)
+              FileUtils.mkdir_p(File.dirname(f_path))
+              zip_file.extract(entry, f_path) unless File.exist?(f_path)
+            end
+          end
+
+          wildcard_path = "#{umls_dir}/20*"
+          Zip::File.open(File.expand_path("#{Dir[wildcard_path][0]}/mmsys.zip")) do |zip_file|
+            zip_file.each do |entry|
+              Inferno.logger.info "Extracting #{entry.name}"
+              f_path = File.join((Dir[wildcard_path][0]).to_s, entry.name)
+              FileUtils.mkdir_p(File.dirname(f_path))
+              zip_file.extract(entry, f_path) unless File.exist?(f_path)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks/validate_code.rb

@@ -0,0 +1,36 @@
+require 'colorize'
+require_relative '../terminology_validation'
+require_relative '../loader'
+
+module Inferno
+  module Terminology
+    module Tasks
+      class ValidateCode
+        include TerminologyValidation
+
+        attr_reader :system, :code, :value_set_url
+
+        def initialize(code:, system:, valueset:)
+          @code = code
+          @system = system
+          @value_set_url = valueset
+        end
+
+        def run
+          Inferno::Terminology::Loader.load_validators
+          code_display = self.system ? "#{self.system}|#{code}" : code.to_s
+          if validate_code(code: code, system: self.system, value_set_url: value_set_url)
+            in_system = 'is in'
+            symbol = "\u2713".encode('utf-8').to_s.green
+          else
+            in_system = 'is not in'
+            symbol = 'X'.red
+          end
+          system_checked = value_set_url || self.system
+
+          Inferno.logger.info "#{symbol} #{code_display} #{in_system} #{system_checked}"
+        end
+      end
+    end
+  end
+end

data/lib/inferno/terminology/tasks.rb

@@ -0,0 +1,11 @@
+Dir.glob(File.join(__dir__, 'tasks', '*.rb')).each do |path|
+  require_relative path.delete_prefix("#{__dir__}/")
+end
+
+module Inferno
+  module Terminology
+    module Tasks
+      TEMP_DIR = 'tmp/terminology'.freeze
+    end
+  end
+end

data/lib/inferno/terminology/terminology_configuration.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Inferno
+  class TerminologyConfiguration
+    class << self
+      def config
+        @config =
+          if File.file? File.join('config', 'terminology_config.yml')
+            YAML.load_file(File.join('config', 'terminology_config.yml')).presence || {}
+          else
+            {}
+          end
+      end
+
+      def allowed_systems_metadata
+        @allowed_systems_metadata ||=
+          Terminology.code_system_metadata
+            .select { |url, _metadata| system_allowed?(url) }
+      end
+
+      def prohibited_systems
+        @prohibited_systems ||=
+          Terminology.code_system_metadata
+            .reject { |url, _metadata| system_allowed?(url) }
+            .keys
+      end
+
+      def prohibited_license_restriction_levels
+        config[:exclude_license_restriction_levels] || []
+      end
+
+      def explicitly_allowed_systems
+        config[:include] || []
+      end
+
+      def explicitly_prohibited_systems
+        config[:exclude] || []
+      end
+
+      def system_allowed?(url)
+        return true if explicitly_allowed_systems.include?(url)
+        return false if explicitly_prohibited_systems.include?(url)
+
+        !prohibited_license_restriction_levels.include?(Terminology.code_system_metadata.dig(url, :restriction_level))
+      end
+
+      def system_prohibited?(url)
+        !system_allowed?(url)
+      end
+    end
+  end
+end

data/lib/inferno/terminology/terminology_validation.rb

@@ -0,0 +1,42 @@
+require_relative '../repositiories/validators'
+require_relative '../terminology/bcp_13'
+
+module Inferno
+  module Terminology
+    module TerminologyValidation
+      # Code systems to "preprocess" prior to validation, and the function to use
+      PREPROCESS_FUNCS = {
+        'urn:ietf:bcp:13' => BCP13.method(:preprocess_code)
+      }.freeze
+
+      def validators_repo
+        @validators_repo ||= Repositories::Validators.new
+      end
+
+      # This function accepts a valueset URL, code, and optional system, and returns true
+      # if the code or code/system combination is valid for the valueset
+      # represented by that URL
+      #
+      # @param String value_set_url the URL for the valueset to validate against
+      # @param String code the code to validate against the valueset
+      # @param String system an optional codesystem to validate against. Defaults to nil
+      # @return Boolean whether the code or code/system is in the valueset
+      def validate_code(code:, value_set_url: nil, system: nil)
+        # Before we validate the code, see if there's any preprocessing steps we have to do
+        # To get the code "ready" for validation
+        code = PREPROCESS_FUNCS[system].call(code) if PREPROCESS_FUNCS[system]
+
+        # Get the valueset from the url. Redundant if the 'system' is not nil,
+        # but allows us to throw a better error if the valueset isn't known by Inferno
+        validator =
+          if value_set_url
+            validators_repo.find(value_set_url) || raise(UnknownValueSetException, value_set_url)
+          else
+            validators_repo.find(system) || raise(UnknownCodeSystemException, system)
+          end
+
+        validator.validate(code: code, system: system)
+      end
+    end
+  end
+end

data/lib/inferno/terminology/validator.rb

@@ -0,0 +1,64 @@
+require_relative 'terminology_configuration'
+
+module Inferno
+  module Terminology
+    class Validator
+      attr_reader :url, :concept_count, :type, :code_systems, :file_name, :bloom_filter
+
+      def initialize(**params)
+        @url = params[:url]
+        @concept_count = params[:count]
+        @type = params[:type]
+        @code_systems = params[:code_systems]
+        @file_name = params[:file]
+        @bloom_filter = params[:bloom_filter]
+      end
+
+      def validate(code:, system: nil)
+        if system
+          raise ProhibitedSystemException, system if TerminologyConfiguration.system_prohibited?(system)
+
+          coding_in_filter?(code: code, system: system)
+        elsif contains_prohibited_systems?
+          raise ProhibitedSystemException, prohibited_systems.join(', ') unless code_in_allowed_system?(code)
+
+          true
+        else
+          code_in_any_system?(code)
+        end
+      end
+
+      def contains_prohibited_systems?
+        prohibited_systems.present?
+      end
+
+      def prohibited_systems
+        @prohibited_systems ||=
+          code_systems.select { |system| TerminologyConfiguration.system_prohibited?(system) }
+      end
+
+      def allowed_systems
+        @allowed_systems ||=
+          code_systems.select { |system| TerminologyConfiguration.system_allowed?(system) }
+      end
+
+      def code_in_allowed_system?(code)
+        code_in_systems?(code, allowed_systems)
+      end
+
+      def code_in_any_system?(code)
+        code_in_systems?(code, code_systems)
+      end
+
+      def code_in_systems?(code, possible_systems)
+        possible_systems.any? do |possible_system|
+          coding_in_filter?(code: code, system: possible_system)
+        end
+      end
+
+      def coding_in_filter?(code:, system:)
+        bloom_filter.include? "#{system}|#{code}"
+      end
+    end
+  end
+end