his_emr_api_lab 1.0.5 → 1.1.0

data/app/services/lab/lims/api/ws_api.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require 'socket.io-client-simple'
+
+module Lab
+  module Lims
+    module Api
+      ##
+      # Retrieve results from LIMS only through a websocket
+      class WsApi
+        def initialize(config)
+          @config = config
+          @results_queue = []
+          @socket = nil
+        end
+
+        def consume_orders(**_kwargs)
+          loop do
+            results = fetch_results
+            unless results
+              Rails.logger.debug('No results available... Waiting for results...')
+              sleep(Lab::Lims::Config.updates_poll_frequency)
+              next
+            end
+
+            Rails.logger.info("Received result for ##{results['tracking_number']}")
+            order = find_order(results['tracking_number'])
+            next unless order
+
+            Rails.logger.info("Updating result for order ##{order.order_id}")
+            yield make_order_dto(order, results), OpenStruct.new(last_seq: 1)
+          end
+        end
+
+        private
+
+        def initialize_socket
+          Rails.logger.debug('Establishing connection to socket...')
+          socket = SocketIO::Client::Simple.connect(socket_url)
+          socket.on(:connect, &method(:on_socket_connect))
+          socket.on(:disconnect, &method(:on_socket_disconnect))
+          socket.on(:results, &method(:on_results_received))
+        end
+
+        def socket_url
+          @config.fetch('url')
+        end
+
+        def on_socket_connect
+          Rails.logger.debug('Connection to LIMS results socket established...')
+        end
+
+        def on_socket_disconnect
+          Rails.logger.debug('Connection to LIMS results socket lost...')
+          @socket = nil
+        end
+
+        def on_results_received(result)
+          Rails.logger.debug("Received result from LIMS: #{result}")
+          tracking_number = result['tracking_number']
+
+          Rails.logger.debug("Queueing result for order ##{tracking_number}")
+          @results_queue.push(result)
+        end
+
+        def order_exists?(tracking_number)
+          Rails.logger.debug("Looking for order for result ##{tracking_number}")
+          orders = OrdersSearchService.find_orders_without_results
+                                      .where(accession_number: tracking_number)
+          # The following ensures that the order was previously pushed to LIMS
+          # or was received from LIMS
+          Lab::LimsOrderMapping.where.not(order: orders).exists?
+        end
+
+        def fetch_results
+          loop do
+            @socket ||= initialize_socket
+
+            results = @results_queue.shift
+            return nil unless results
+
+            unless order_exists?(results['tracking_number'])
+              Rails.logger.debug("Ignoring result for order ##{tracking_number}")
+              next
+            end
+
+            return results
+          end
+        end
+
+        def find_order(lims_id)
+          mapping = Lab::LimsOrderMapping.where(lims_id: lims_id).select(:order_id)
+          Lab::LabOrder.find_by(order_id: mapping)
+        end
+
+        def make_order_dto(order, results)
+          Lab::Lims::OrderSerializer
+            .serialize_order(order)
+            .merge(
+              id: order.accession_number,
+              test_results: {
+                results['test_name'] => {
+                  results: results['results'].each_with_object({}) do |measure, formatted_measures|
+                    measure_name, measure_value = measure
+
+                    formatted_measures[measure_name] = { result_value: measure_value }
+                  end,
+                  result_date: results['date_updated'],
+                  result_entered_by: {
+                    first_name: results['who_updated']['first_name'],
+                    last_name: results['who_updated']['last_name'],
+                    id: results['who_updated']['id_number']
+                  }
+                }
+              }
+            )
+        end
+      end
+    end
+  end
+end

data/app/services/lab/lims/config.rb

@@ -5,6 +5,9 @@ module Lab
     ##
     # Load LIMS' configuration files
     module Config
+      # TODO: Replace this maybe with `Rails.application.configuration.lab.lims`
+      # so that we do not have to directly mess with configuration files.
+
       class ConfigNotFound < RuntimeError; end
 
       class << self
@@ -12,31 +15,63 @@ module Lab
         # Returns LIMS' couchdb configuration file for the current environment (Rails.env)
         def couchdb
           config_path = begin
-            find_config_path('couchdb.yml')
-          rescue ConfigNotFound => e
-            Rails.logger.error("Failed to find default LIMS couchdb config: #{e.message}")
-            find_config_path('couchdb-lims.yml') # This can be placed in HIS-EMR-API/config
-          end
+                          find_config_path('couchdb.yml')
+                        rescue ConfigNotFound => e
+                          Rails.logger.error("Failed to find default LIMS couchdb config: #{e.message}")
+                          find_config_path('couchdb-lims.yml') # This can be placed in HIS-EMR-API/config
+                        end
 
           Rails.logger.debug("Using LIMS couchdb config: #{config_path}")
 
           YAML.load_file(config_path)[Rails.env]
         end
 
+        def rest_api
+          @rest_api ||= {
+            protocol: emr_api_application('lims_protocol', 'http'),
+            host: emr_api_application('lims_host'),
+            port: emr_api_application('lims_port'),
+            username: emr_api_application('lims_username'),
+            password: emr_api_application('lims_password')
+          }
+        end
+
+        def updates_socket
+          @updates_socket ||= {
+            'url' => emr_api_application('lims_realtime_updates_url')
+          }
+        end
+
+        def updates_poll_frequency
+          30 # Seconds
+        end
+
         ##
         # Returns LIMS' application.yml configuration file
         def application
-          YAML.load_file(find_config_path('application.yml'))
+          @application ||= YAML.load_file(find_config_path('application.yml'))
         end
 
         ##
         # Returns LIMS' database.yml configuration file
         def database
-          YAML.load_file(find_config_path('database.yml'))
+          @database ||= YAML.load_file(find_config_path('database.yml'))
         end
 
         private
 
+        def emr_api_application(param, fallback = nil)
+          @emr_api_application ||= YAML.load_file(Rails.root.join('config', 'application.yml'))
+
+          @emr_api_application.fetch(param) do
+            unless fallback
+              raise ConfigNotFound, "Missing config param: #{param}"
+            end
+
+            fallback
+          end
+        end
+
         ##
         # Looks for a config file in various LIMS installation directories
         #
@@ -48,7 +83,9 @@ module Lab
             Rails.root.parent.join("nlims_controller/config/#{filename}")
           ]
 
-          paths = [Rails.root.join('config/lims-couchdb.yml'), *paths] if filename == 'couchdb.yml'
+          if filename == 'couchdb.yml'
+            paths = [Rails.root.join('config/lims-couchdb.yml'), *paths]
+          end
 
           paths.each do |path|
             Rails.logger.debug("Looking for LIMS couchdb config at: #{path}")

data/app/services/lab/lims/migrator.rb

@@ -29,8 +29,11 @@ require 'lab/lab_test'
 require 'lab/lims_order_mapping'
 require 'lab/lims_failed_import'
 
+require_relative './api/couchdb_api'
 require_relative './config'
-require_relative './worker'
+require_relative './pull_worker'
+require_relative './utils'
+
 require_relative '../orders_service'
 require_relative '../results_service'
 require_relative '../tests_service'
@@ -46,7 +49,7 @@ module Lab
     module Migrator
       MAX_THREADS = ENV.fetch('MIGRATION_WORKERS', 6).to_i
 
-      class CouchDbMigratorApi < Api::CouchDbApi
+      class CouchDbMigratorApi < Lab::Lims::Api::CouchDbApi
         def initialize(*args, processes: 1, on_merge_processes: nil, **kwargs)
           super(*args, **kwargs)
 
@@ -88,8 +91,8 @@ module Lab
         end
       end
 
-      class MigrationWorker < Worker
-        LOG_FILE_PATH = LIMS_LOG_PATH.join('migration-last-id.dat')
+      class MigrationWorker < PullWorker
+        LOG_FILE_PATH = Utils::LIMS_LOG_PATH.join('migration-last-id.dat')
 
         attr_reader :rejections
 
@@ -132,7 +135,8 @@ module Lab
         end
       end
 
-      MIGRATION_REJECTIONS_CSV_PATH = LIMS_LOG_PATH.join('migration-rejections.csv')
+      # NOTE: LIMS_LOG_PATH below is defined in worker.rb
+      MIGRATION_REJECTIONS_CSV_PATH = Utils::LIMS_LOG_PATH.join('migration-rejections.csv')
 
       def self.export_rejections(rejections)
         headers = ['doc_id', 'Accession number', 'NHID', 'First name', 'Last name', 'Reason']
@@ -150,7 +154,7 @@ module Lab
         save_csv(MIGRATION_REJECTIONS_CSV_PATH, headers: headers, rows: rows)
       end
 
-      MIGRATION_FAILURES_CSV_PATH = LIMS_LOG_PATH.join('migration-failures.csv')
+      MIGRATION_FAILURES_CSV_PATH = Utils::LIMS_LOG_PATH.join('migration-failures.csv')
 
       def self.export_failures
         headers = ['doc_id', 'Accession number', 'NHID', 'Reason', 'Difference']
@@ -167,10 +171,10 @@ module Lab
         save_csv(MIGRATION_FAILURES_CSV_PATH, headers: headers, rows: rows)
       end
 
-      MIGRATION_LOG_PATH = LIMS_LOG_PATH.join('migration.log')
+      MIGRATION_LOG_PATH = Utils::LIMS_LOG_PATH.join('migration.log')
 
       def self.start_migration
-        Dir.mkdir(LIMS_LOG_PATH) unless File.exist?(LIMS_LOG_PATH)
+        Dir.mkdir(Utils::LIMS_LOG_PATH) unless File.exist?(Utils::LIMS_LOG_PATH)
 
         logger = LoggerMultiplexor.new(Logger.new($stdout), MIGRATION_LOG_PATH)
         logger.level = :debug
@@ -185,7 +189,6 @@ module Lab
                     end
 
         worker = MigrationWorker.new(api_class)
-
         worker.pull_orders(batch_size: 10_000)
       ensure
         worker && export_rejections(worker.rejections)

data/app/services/lab/lims/order_serializer.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative './config'
 require_relative './order_dto'
 require_relative './utils'
 
@@ -12,9 +13,9 @@ module Lab
         include Utils
 
         def serialize_order(order)
-          serialized_order = Utils.structify(Lab::LabOrderSerializer.serialize_order(order))
+          serialized_order = Lims::Utils.structify(Lab::LabOrderSerializer.serialize_order(order))
 
-          OrderDTO.new(
+          Lims::OrderDTO.new(
             tracking_number: serialized_order.accession_number,
             sending_facility: current_facility_name,
             receiving_facility: serialized_order.target_lab,
@@ -69,7 +70,11 @@ module Lab
         end
 
         def format_sample_type(name)
-          name.casecmp?('Unknown') ? 'not_specified' : name.titleize
+          return 'not_specified' if name.casecmp?('Unknown')
+
+          return 'CSF' if name.casecmp?('Cerebrospinal Fluid')
+
+          name.titleize
         end
 
         def format_sample_status(name)
@@ -77,7 +82,9 @@ module Lab
         end
 
         def format_sample_status_trail(order)
-          return [] if order.concept_id == ConceptName.find_by_name!('Unknown').concept_id
+          if order.concept_id == ConceptName.find_by_name!('Unknown').concept_id
+            return []
+          end
 
           user = User.find(order.discontinued_by || order.creator)
           drawn_by = PersonName.find_by_person_id(user.user_id)
@@ -97,7 +104,9 @@ module Lab
         end
 
         def format_test_status_trail(order)
-          order.tests.each_with_object({}) do |test, trail|
+          tests = order.voided.zero? ? order.tests : Lab::LabOrderSerializer.voided_tests(order)
+
+          tests.each_with_object({}) do |test, trail|
             test_name = format_test_name(ConceptName.find_by_concept_id!(test.value_coded).name)
 
             current_test_trail = trail[test_name] = {}
@@ -107,6 +116,13 @@ module Lab
               updated_by: find_user(test.creator)
             }
 
+            unless test.voided.zero?
+              current_test_trail[test.date_voided.strftime('%Y%m%d%H%M%S')] = {
+                status: 'Voided',
+                updated_by: find_user(test.voided_by)
+              }
+            end
+
             next unless test.result
 
             current_test_trail[test.obs_datetime.strftime('%Y%m%d%H%M%S')] = {
@@ -122,7 +138,10 @@ module Lab
 
         def format_test_results(order)
           order.tests&.each_with_object({}) do |test, results|
-            next unless test.result
+            next if test.result.nil? || test.result.empty?
+
+            test_creator = User.find(Observation.find(test.result.first.id).creator)
+            test_creator_name = PersonName.find_by_person_id(test_creator.person_id)
 
             results[format_test_name(test.name)] = {
               results: test.result.each_with_object({}) do |measure, measures|
@@ -131,7 +150,11 @@ module Lab
                 }
               end,
               result_date: test.result.first&.date,
-              result_entered_by: {}
+              result_entered_by: {
+                first_name: test_creator_name&.given_name,
+                last_name: test_creator_name&.family_name,
+                id: test_creator.username
+              }
             }
           end
         end
@@ -165,7 +188,7 @@ module Lab
           return district if district
 
           GlobalProperty.create(property: 'current_health_center_district',
-                                property_value: Config.application['district'],
+                                property_value: Lims::Config.application['district'],
                                 uuid: SecureRandom.uuid)
 
           Config.application['district']

data/app/services/lab/lims/pull_worker.rb

@@ -0,0 +1,295 @@
+# frozen_string_literal: true
+
+module Lab
+  module Lims
+    class PullWorker
+      attr_reader :lims_api
+
+      include Utils # for logger
+
+      LIMS_LOG_PATH = Rails.root.join('log', 'lims')
+
+      def initialize(lims_api)
+        @lims_api = lims_api
+      end
+
+      ##
+      # Pulls orders from the LIMS queue and writes them to the local database
+      def pull_orders(batch_size: 10_000, **kwargs)
+        logger.info("Retrieving LIMS orders starting from #{last_seq}")
+
+        lims_api.consume_orders(from: last_seq, limit: batch_size, **kwargs) do |order_dto, context|
+          logger.debug("Retrieved order ##{order_dto[:tracking_number]}: #{order_dto}")
+
+          patient = find_patient_by_nhid(order_dto[:patient][:id])
+          unless patient
+            logger.debug("Discarding order: Non local patient ##{order_dto[:patient][:id]} on order ##{order_dto[:tracking_number]}")
+            order_rejected(order_dto, "Patient NPID, '#{order_dto[:patient][:id]}', didn't match any local NPIDs")
+            next
+          end
+
+          if order_dto[:tests].empty?
+            logger.debug("Discarding order: Missing tests on order ##{order_dto[:tracking_number]}")
+            order_rejected(order_dto, 'Order is missing tests')
+            next
+          end
+
+          diff = match_patient_demographics(patient, order_dto['patient'])
+          if diff.empty?
+            save_order(patient, order_dto)
+            order_saved(order_dto)
+          else
+            save_failed_import(order_dto, 'Demographics not matching', diff)
+          end
+
+          update_last_seq(context.current_seq)
+        rescue DuplicateNHID
+          logger.warn("Failed to import order due to duplicate patient NHID: #{order_dto[:patient][:id]}")
+          save_failed_import(order_dto, "Duplicate local patient NHID: #{order_dto[:patient][:id]}")
+        rescue MissingAccessionNumber
+          logger.warn("Failed to import order due to missing accession number: #{order_dto[:_id]}")
+          save_failed_import(order_dto, 'Order missing tracking number')
+        rescue LimsException => e
+          logger.warn("Failed to import order due to #{e.class} - #{e.message}")
+          save_failed_import(order_dto, e.message)
+        end
+      end
+
+      protected
+
+      def order_saved(order_dto); end
+
+      def order_rejected(order_dto, message); end
+
+      def last_seq
+        File.open(last_seq_path, File::RDONLY | File::CREAT, 0o644) do |fin|
+          data = fin.read&.strip
+          return nil if data.blank?
+
+          return data
+        end
+      end
+
+      def update_last_seq(last_seq)
+        File.open(last_seq_path, File::WRONLY | File::CREAT, 0o644) do |fout|
+          fout.flock(File::LOCK_EX)
+
+          fout.write(last_seq.to_s)
+        end
+      end
+
+      private
+
+      def find_patient_by_nhid(nhid)
+        national_id_type = PatientIdentifierType.where(name: ['National id', 'Old Identification Number'])
+        identifiers = PatientIdentifier.where(type: national_id_type, identifier: nhid)
+                                       .joins('INNER JOIN person ON person.person_id = patient_identifier.patient_id AND person.voided  = 0')
+        if identifiers.count.zero?
+          identifiers = PatientIdentifier.unscoped
+                                         .where(voided: 1, type: national_id_type, identifier: nhid)
+                                         .joins('INNER JOIN person ON person.person_id = patient_identifier.patient_id AND person.voided  = 0')
+        end
+
+        # Joining to person above to ensure that the person is not voided,
+        # it was noted at one site that there were some people that were voided
+        # upon merging but the patient and patient_identifier was not voided
+
+        return nil if identifiers.count.zero?
+
+        patients = Patient.where(patient_id: identifiers.select(:patient_id))
+                          .distinct(:patient_id)
+                          .all
+
+        if patients.size > 1
+          raise DuplicateNHID, "Duplicate National Health ID: #{nhid}"
+        end
+
+        patients.first
+      end
+
+      ##
+      # Matches a local patient's demographics to a LIMS patient's demographics
+      def match_patient_demographics(local_patient, lims_patient)
+        diff = {}
+        person = Person.find(local_patient.id)
+        person_name = PersonName.find_by_person_id(local_patient.id)
+
+        unless (person.gender.blank? && lims_patient['gender'].blank?)\
+          || person.gender&.first&.casecmp?(lims_patient['gender']&.first)
+          diff[:gender] = { local: person.gender, lims: lims_patient['gender'] }
+        end
+
+        unless names_match?(person_name&.given_name, lims_patient['first_name'])
+          diff[:given_name] = { local: person_name&.given_name, lims: lims_patient['first_name'] }
+        end
+
+        unless names_match?(person_name&.family_name, lims_patient['last_name'])
+          diff[:family_name] = { local: person_name&.family_name, lims: lims_patient['last_name'] }
+        end
+
+        diff
+      end
+
+      def names_match?(name1, name2)
+        name1 = name1&.gsub(/'/, '')&.strip
+        name2 = name2&.gsub(/'/, '')&.strip
+
+        return true if name1.blank? && name2.blank?
+
+        return false if name1.blank? || name2.blank?
+
+        name1.casecmp?(name2)
+      end
+
+      def save_order(patient, order_dto)
+        raise MissingAccessionNumber if order_dto[:tracking_number].blank?
+
+        logger.info("Importing LIMS order ##{order_dto[:tracking_number]}")
+        mapping = find_order_mapping_by_lims_id(order_dto[:_id])
+
+        ActiveRecord::Base.transaction do
+          if mapping
+            order = update_order(patient, mapping.order_id, order_dto)
+            mapping.update(pulled_at: Time.now)
+          else
+            order = create_order(patient, order_dto)
+            mapping = LimsOrderMapping.create(lims_id: order_dto[:_id],
+                                              order_id: order['id'],
+                                              pulled_at: Time.now,
+                                              revision: order_dto['_rev'])
+          end
+
+          order
+        end
+      end
+
+      def create_order(patient, order_dto)
+        logger.debug("Creating order ##{order_dto['_id']}")
+        order = OrdersService.order_test(order_dto.to_order_service_params(patient_id: patient.patient_id))
+        unless order_dto['test_results'].empty?
+          update_results(order, order_dto['test_results'])
+        end
+
+        order
+      end
+
+      def update_order(patient, order_id, order_dto)
+        logger.debug("Updating order ##{order_dto['_id']}")
+        order = OrdersService.update_order(order_id, order_dto.to_order_service_params(patient_id: patient.patient_id)
+                                                              .merge(force_update: true))
+        unless order_dto['test_results'].empty?
+          update_results(order, order_dto['test_results'])
+        end
+
+        order
+      end
+
+      def update_results(order, lims_results)
+        logger.debug("Updating results for order ##{order[:accession_number]}: #{lims_results}")
+
+        lims_results.each do |test_name, test_results|
+          test = find_test(order['id'], test_name)
+          unless test
+            logger.warn("Couldn't find test, #{test_name}, in order ##{order[:id]}")
+            next
+          end
+
+          next unless test_results['results']
+
+          measures = test_results['results'].map do |indicator, value|
+            measure = find_measure(order, indicator, value)
+            next nil unless measure
+
+            measure
+          end
+
+          measures = measures.compact
+          next if measures.empty?
+
+          creator = format_result_entered_by(test_results['result_entered_by'])
+
+          ResultsService.create_results(test.id, provider_id: User.current.person_id,
+                                                 date: Utils.parse_date(test_results['date_result_entered'], order[:order_date].to_s),
+                                                 comments: "LIMS import: Entered by: #{creator}",
+                                                 measures: measures)
+        end
+      end
+
+      def find_test(order_id, test_name)
+        test_name = Utils.translate_test_name(test_name)
+        test_concept = Utils.find_concept_by_name(test_name)
+        raise "Unknown test name, #{test_name}!" unless test_concept
+
+        LabTest.find_by(order_id: order_id, value_coded: test_concept.concept_id)
+      end
+
+      def find_measure(_order, indicator_name, value)
+        indicator = Utils.find_concept_by_name(indicator_name)
+        unless indicator
+          logger.warn("Result indicator #{indicator_name} not found in concepts list")
+          return nil
+        end
+
+        value_modifier, value, value_type = parse_lims_result_value(value)
+        return nil if value.blank?
+
+        ActiveSupport::HashWithIndifferentAccess.new(
+          indicator: { concept_id: indicator.concept_id },
+          value_type: value_type,
+          value: value_type == 'numeric' ? value.to_f : value,
+          value_modifier: value_modifier.blank? ? '=' : value_modifier
+        )
+      end
+
+      def parse_lims_result_value(value)
+        value = value['result_value']&.strip
+        return nil, nil, nil if value.blank?
+
+        match = value&.match(/^(>|=|<|<=|>=)(.*)$/)
+        return nil, value, guess_result_datatype(value) unless match
+
+        [match[1], match[2].strip, guess_result_datatype(match[2])]
+      end
+
+      def guess_result_datatype(result)
+        return 'numeric' if result.strip.match?(/^[+-]?((\d+(\.\d+)?)|\.\d+)$/)
+
+        'text'
+      end
+
+      def format_result_entered_by(result_entered_by)
+        first_name = result_entered_by['first_name']
+        last_name = result_entered_by['last_name']
+        phone_number = result_entered_by['phone_number']
+        id = result_entered_by['id'] # Looks like a user_id of some sort
+
+        "#{id}:#{first_name} #{last_name}:#{phone_number}"
+      end
+
+      def save_failed_import(order_dto, reason, diff = nil)
+        logger.info("Failed to import LIMS order ##{order_dto[:tracking_number]} due to '#{reason}'")
+        LimsFailedImport.create!(lims_id: order_dto[:_id],
+                                 tracking_number: order_dto[:tracking_number],
+                                 patient_nhid: order_dto[:patient][:id],
+                                 reason: reason,
+                                 diff: diff&.to_json)
+      end
+
+      def last_seq_path
+        LIMS_LOG_PATH.join('last_seq.dat')
+      end
+
+      def find_order_mapping_by_lims_id(lims_id)
+        mapping = Lab::LimsOrderMapping.find_by(lims_id: lims_id)
+        return nil unless mapping
+
+        if Lab::LabOrder.where(order_id: mapping.order_id).exists?
+          return mapping
+        end
+
+        mapping.destroy
+        nil
+      end
+    end
+  end
+end