his_emr_api_lab 1.1.22 → 1.1.23

data/app/services/lab/lims/pull_worker.rb

@@ -0,0 +1,289 @@
+# frozen_string_literal: true
+
+module Lab
+  module Lims
+    ##
+    # Pulls orders from a Lims API object and saves them to the local database.
+    class PullWorker
+      attr_reader :lims_api
+
+      include Utils # for logger
+
+      LIMS_LOG_PATH = Rails.root.join('log', 'lims')
+
+      def initialize(lims_api)
+        @lims_api = lims_api
+      end
+
+      ##
+      # Pulls orders from the LIMS queue and writes them to the local database
+      def pull_orders(batch_size: 10_000, **kwargs)
+        logger.info("Retrieving LIMS orders starting from #{last_seq}")
+
+        lims_api.consume_orders(from: last_seq, limit: batch_size, **kwargs) do |order_dto, context|
+          logger.debug("Retrieved order ##{order_dto[:tracking_number]}: #{order_dto}")
+
+          patient = find_patient_by_nhid(order_dto[:patient][:id])
+          unless patient
+            logger.debug("Discarding order: Non local patient ##{order_dto[:patient][:id]} on order ##{order_dto[:tracking_number]}")
+            order_rejected(order_dto, "Patient NPID, '#{order_dto[:patient][:id]}', didn't match any local NPIDs")
+            next
+          end
+
+          if order_dto[:tests].empty?
+            logger.debug("Discarding order: Missing tests on order ##{order_dto[:tracking_number]}")
+            order_rejected(order_dto, 'Order is missing tests')
+            next
+          end
+
+          diff = match_patient_demographics(patient, order_dto['patient'])
+          if diff.empty?
+            save_order(patient, order_dto)
+            order_saved(order_dto)
+          else
+            save_failed_import(order_dto, 'Demographics not matching', diff)
+          end
+
+          update_last_seq(context.current_seq)
+        rescue Lab::Lims::DuplicateNHID
+          logger.warn("Failed to import order due to duplicate patient NHID: #{order_dto[:patient][:id]}")
+          save_failed_import(order_dto, "Duplicate local patient NHID: #{order_dto[:patient][:id]}")
+        rescue MissingAccessionNumber
+          logger.warn("Failed to import order due to missing accession number: #{order_dto[:_id]}")
+          save_failed_import(order_dto, 'Order missing tracking number')
+        rescue LimsException => e
+          logger.warn("Failed to import order due to #{e.class} - #{e.message}")
+          save_failed_import(order_dto, e.message)
+        end
+      end
+
+      protected
+
+      def order_saved(order_dto); end
+
+      def order_rejected(order_dto, message); end
+
+      def last_seq
+        File.open(last_seq_path, File::RDONLY | File::CREAT, 0o644) do |fin|
+          data = fin.read&.strip
+          return nil if data.blank?
+
+          return data
+        end
+      end
+
+      def update_last_seq(last_seq)
+        File.open(last_seq_path, File::WRONLY | File::CREAT, 0o644) do |fout|
+          fout.flock(File::LOCK_EX)
+
+          fout.write(last_seq.to_s)
+        end
+      end
+
+      private
+
+      def find_patient_by_nhid(nhid)
+        national_id_type = PatientIdentifierType.where(name: ['National id', 'Old Identification Number'])
+        identifiers = PatientIdentifier.where(type: national_id_type, identifier: nhid)
+                                       .joins('INNER JOIN person ON person.person_id = patient_identifier.patient_id AND person.voided  = 0')
+        if identifiers.count.zero?
+          identifiers = PatientIdentifier.unscoped
+                                         .where(voided: 1, type: national_id_type, identifier: nhid)
+                                         .joins('INNER JOIN person ON person.person_id = patient_identifier.patient_id AND person.voided  = 0')
+        end
+
+        # Joining to person above to ensure that the person is not voided,
+        # it was noted at one site that there were some people that were voided
+        # upon merging but the patient and patient_identifier was not voided
+
+        return nil if identifiers.count.zero?
+
+        patients = Patient.where(patient_id: identifiers.select(:patient_id))
+                          .distinct(:patient_id)
+                          .all
+
+        raise Lab::Lims::DuplicateNHID, "Duplicate National Health ID: #{nhid}" if patients.size > 1
+
+        patients.first
+      end
+
+      ##
+      # Matches a local patient's demographics to a LIMS patient's demographics
+      def match_patient_demographics(local_patient, lims_patient)
+        diff = {}
+        person = Person.find(local_patient.id)
+        person_name = PersonName.find_by_person_id(local_patient.id)
+
+        unless (person.gender.blank? && lims_patient['gender'].blank?)\
+          || person.gender&.first&.casecmp?(lims_patient['gender']&.first)
+          diff[:gender] = { local: person.gender, lims: lims_patient['gender'] }
+        end
+
+        unless names_match?(person_name&.given_name, lims_patient['first_name'])
+          diff[:given_name] = { local: person_name&.given_name, lims: lims_patient['first_name'] }
+        end
+
+        unless names_match?(person_name&.family_name, lims_patient['last_name'])
+          diff[:family_name] = { local: person_name&.family_name, lims: lims_patient['last_name'] }
+        end
+
+        diff
+      end
+
+      def names_match?(name1, name2)
+        name1 = name1&.gsub(/'/, '')&.strip
+        name2 = name2&.gsub(/'/, '')&.strip
+
+        return true if name1.blank? && name2.blank?
+
+        return false if name1.blank? || name2.blank?
+
+        name1.casecmp?(name2)
+      end
+
+      def save_order(patient, order_dto)
+        raise MissingAccessionNumber if order_dto[:tracking_number].blank?
+
+        logger.info("Importing LIMS order ##{order_dto[:tracking_number]}")
+        mapping = find_order_mapping_by_lims_id(order_dto[:_id])
+
+        ActiveRecord::Base.transaction do
+          if mapping
+            order = update_order(patient, mapping.order_id, order_dto)
+            mapping.update(pulled_at: Time.now)
+          else
+            order = create_order(patient, order_dto)
+            mapping = LimsOrderMapping.create(lims_id: order_dto[:_id],
+                                              order_id: order['id'],
+                                              pulled_at: Time.now,
+                                              revision: order_dto['_rev'])
+          end
+
+          order
+        end
+      end
+
+      def create_order(patient, order_dto)
+        logger.debug("Creating order ##{order_dto['_id']}")
+        order = OrdersService.order_test(order_dto.to_order_service_params(patient_id: patient.patient_id))
+        update_results(order, order_dto['test_results']) unless order_dto['test_results'].empty?
+
+        order
+      end
+
+      def update_order(patient, order_id, order_dto)
+        logger.debug("Updating order ##{order_dto['_id']}")
+        order = OrdersService.update_order(order_id, order_dto.to_order_service_params(patient_id: patient.patient_id)
+                                                              .merge(force_update: 'true'))
+        update_results(order, order_dto['test_results']) unless order_dto['test_results'].empty?
+
+        order
+      end
+
+      def update_results(order, lims_results)
+        logger.debug("Updating results for order ##{order[:accession_number]}: #{lims_results}")
+        
+        lims_results.each do |test_name, test_results|
+          test = find_test(order['id'], test_name)
+          unless test
+            logger.warn("Couldn't find test, #{test_name}, in order ##{order[:id]}")
+            next
+          end
+          
+          next if test.result || test_results['results'].blank?
+
+          measures = test_results['results'].map do |indicator, value|
+            measure = find_measure(order, indicator, value)
+            next nil unless measure
+
+            measure
+          end
+          
+          measures = measures.compact
+          next if measures.empty?
+
+          creator = format_result_entered_by(test_results['result_entered_by'])
+
+          ResultsService.create_results(test.id,  { provider_id: User.current.person_id,
+                                                    date: Utils.parse_date(test_results['date_result_entered'], order[:order_date].to_s),
+                                                    comments: "LIMS import: Entered by: #{creator}",
+                                                    measures: measures } )
+        end
+      end
+
+      def find_test(order_id, test_name)
+        test_name = Utils.translate_test_name(test_name)
+        test_concept = Utils.find_concept_by_name(test_name)
+        raise "Unknown test name, #{test_name}!" unless test_concept
+
+        LabTest.find_by(order_id: order_id, value_coded: test_concept.concept_id)
+      end
+
+      def find_measure(_order, indicator_name, value)
+        indicator = Utils.find_concept_by_name(indicator_name)
+        unless indicator
+          logger.warn("Result indicator #{indicator_name} not found in concepts list")
+          return nil
+        end
+
+        value_modifier, value, value_type = parse_lims_result_value(value)
+        return nil if value.blank?
+
+        ActiveSupport::HashWithIndifferentAccess.new(
+          indicator: { concept_id: indicator.concept_id },
+          value_type: value_type,
+          value: value_type == 'numeric' ? value.to_f : value,
+          value_modifier: value_modifier.blank? ? '=' : value_modifier
+        )
+      end
+
+      def parse_lims_result_value(value)
+        value = value['result_value']&.strip
+        return nil, nil, nil if value.blank?
+
+        match = value&.match(/^(>|=|<|<=|>=)(.*)$/)
+        return nil, value, guess_result_datatype(value) unless match
+
+        [match[1], match[2].strip, guess_result_datatype(match[2])]
+      end
+
+      def guess_result_datatype(result)
+        return 'numeric' if result.strip.match?(/^[+-]?((\d+(\.\d+)?)|\.\d+)$/)
+
+        'text'
+      end
+
+      def format_result_entered_by(result_entered_by)
+        first_name = result_entered_by['first_name']
+        last_name = result_entered_by['last_name']
+        phone_number = result_entered_by['phone_number']
+        id = result_entered_by['id'] # Looks like a user_id of some sort
+
+        "#{id}:#{first_name} #{last_name}:#{phone_number}"
+      end
+
+      def save_failed_import(order_dto, reason, diff = nil)
+        logger.info("Failed to import LIMS order ##{order_dto[:tracking_number]} due to '#{reason}'")
+        LimsFailedImport.create!(lims_id: order_dto[:_id],
+                                 tracking_number: order_dto[:tracking_number],
+                                 patient_nhid: order_dto[:patient][:id],
+                                 reason: reason,
+                                 diff: diff&.to_json)
+      end
+
+      def last_seq_path
+        LIMS_LOG_PATH.join('last_seq.dat')
+      end
+
+      def find_order_mapping_by_lims_id(lims_id)
+        mapping = Lab::LimsOrderMapping.find_by(lims_id: lims_id)
+        return nil unless mapping
+
+        return mapping if Lab::LabOrder.where(order_id: mapping.order_id).exists?
+
+        mapping.destroy
+        nil
+      end
+    end
+  end
+end

data/app/services/lab/lims/push_worker.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Lab
+  module Lims
+    ##
+    # Pushes all local orders to a LIMS Api object.
+    class PushWorker
+      attr_reader :lims_api
+
+      include Utils # for logger
+
+      SECONDS_TO_WAIT_FOR_ORDERS = 30
+
+      def initialize(lims_api)
+        @lims_api = lims_api
+      end
+
+      def push_orders(batch_size: 1000, wait: false)
+        loop do
+          logger.info('Looking for new orders to push to LIMS...')
+          orders = orders_pending_sync(batch_size).all
+
+          logger.debug("Found #{orders.size} orders...")
+          orders.each do |order|
+            push_order(order)
+          rescue GatewayError => e
+            logger.error("Failed to push order ##{order.accession_number}: #{e.class} - #{e.message}")
+          end
+
+          break unless wait
+
+          logger.info('Waiting for orders...')
+          sleep(Lab::Lims::Config.updates_poll_frequency)
+        end
+      end
+
+      def push_order_by_id(order_id)
+        order = Lab::LabOrder.joins(order_type: { name: 'Lab' })
+                             .unscoped
+                             .find(order_id)
+        push_order(order)
+      end
+
+      ##
+      # Pushes given order to LIMS queue
+      def push_order(order)
+        logger.info("Pushing order ##{order.order_id}")
+
+        order_dto = Lab::Lims::OrderSerializer.serialize_order(order)
+        mapping = Lab::LimsOrderMapping.find_by(order_id: order.order_id)
+
+        ActiveRecord::Base.transaction do
+          if mapping && !order.voided.zero?
+            Rails.logger.info("Deleting order ##{order_dto[:accession_number]} from LIMS")
+            lims_api.delete_order(mapping.lims_id, order_dto)
+            mapping.destroy
+          elsif mapping
+            Rails.logger.info("Updating order ##{order_dto[:accession_number]} in LIMS")
+            lims_api.update_order(mapping.lims_id, order_dto)
+            if order_dto['test_results'].nil? || order_dto['test_results'].empty?
+              mapping.update(pushed_at: Time.now)             
+            else
+              mapping.update(pushed_at: Time.now, result_push_status: true)  
+           end
+          elsif order_dto[:_id] && Lab::LimsOrderMapping.where(lims_id: order_dto[:_id]).exists?
+            # HACK: v1.1.7 had a bug where duplicates of recently created orders where being created by
+            # the pull worker. This here detects those duplicates and voids them.
+            Rails.logger.warn("Duplicate accession number found: #{order_dto[:_id]}, skipping order...")
+            fix_duplicates!(order)
+          else
+            Rails.logger.info("Creating order ##{order_dto[:accession_number]} in LIMS")
+            update = lims_api.create_order(order_dto)
+            Lab::LimsOrderMapping.create!(order: order, lims_id: update['id'], revision: update['rev'],
+                                          pushed_at: Time.now, result_push_status: false)
+          end
+        end
+
+        order_dto
+      end
+
+      private
+
+      def orders_pending_sync(batch_size)
+        return new_orders.limit(batch_size) if new_orders.exists?
+
+        return voided_orders.limit(batch_size) if voided_orders.exists?
+
+        updated_orders.limit(batch_size)
+      end
+
+      def new_orders
+        Rails.logger.debug('Looking for new orders that need to be created in LIMS...')
+        Lab::LabOrder.where.not(order_id: Lab::LimsOrderMapping.all.select(:order_id))
+                     .order(date_created: :desc)
+      end
+
+      def updated_orders
+        Rails.logger.debug('Looking for recently updated orders that need to be pushed to LIMS...')
+        last_updated = Lab::LimsOrderMapping.select('MAX(updated_at) AS last_updated')
+                                            .first
+                                            .last_updated
+
+        Lab::LabOrder.left_joins(:results)
+                     .joins(:mapping)
+                     .where('orders.discontinued_date > :last_updated
+                             OR obs.date_created > orders.date_created AND lab_lims_order_mappings.result_push_status = 0',
+                            last_updated: last_updated)
+                     .group('orders.order_id')
+                     .order(discontinued_date: :desc, date_created: :desc)
+      end
+
+      def voided_orders
+        Rails.logger.debug('Looking for voided orders that are being tracked by LIMS...')
+        Lab::LabOrder.unscoped
+                     .where(order_type: OrderType.where(name: Lab::Metadata::ORDER_TYPE_NAME),
+                            order_id: Lab::LimsOrderMapping.all.select(:order_id),
+                            voided: 1)
+                     .order(date_voided: :desc)
+      end
+
+      ##
+      # HACK: Checks for duplicates previously created by version 1.1.7 pull worker bug due to this proving orders
+      # that have not been pushed to LIMS as orders awaiting updates.
+      def fix_duplicates!(order)
+        return order.void('Duplicate created by bug in HIS-EMR-API-Lab v1.1.7') unless order_has_specimen?(order)
+
+        duplicate_order = Lab::LabOrder.where(accession_number: order.accession_number)
+                                       .where.not(order_id: order.order_id)
+                                       .first
+        return unless duplicate_order
+
+        if !order_has_results?(order) && (order_has_results?(duplicate_order) || order_has_specimen?(duplicate_order))
+          order.void('DUplicate created by bug in HIS-EMR-API-Lab v1.1.7')
+        else
+          duplicate_order.void('Duplicate created by bug in HIS-EMR-API-Lab v1.1.7')
+          Lab::LimsOrderMapping.find_by_lims_id(order.accession_number)&.destroy
+        end
+      end
+
+      def order_has_results?(order)
+        order.results.exists?
+      end
+
+      def order_has_specimen?(order)
+        order.concept_id == ConceptName.find_by_name!('Unknown').concept_id
+      end
+    end
+  end
+end

data/app/services/lab/lims/utils.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'cgi/util'
+
+module Lab
+  module Lims
+    ##
+    # Various helper methods for modules in the Lims namespaces...
+    module Utils
+      LIMS_LOG_PATH = Rails.root.join('log', 'lims')
+      FileUtils.mkdir_p(LIMS_LOG_PATH) unless File.exist?(LIMS_LOG_PATH)
+
+      def logger
+        Rails.logger
+      end
+
+      TEST_NAME_MAPPINGS = {
+        # For some weird reason(s) some tests have multiple names in LIMS,
+        # this is used to sanitize those names.
+        'hiv_viral_load' => 'HIV Viral Load',
+        'viral laod' => 'HIV Viral Load',
+        'viral load' => 'HIV Viral Load',
+        'i/ink' => 'India ink',
+        'indian ink' => 'India ink'
+      }.freeze
+
+      def self.translate_test_name(test_name)
+        TEST_NAME_MAPPINGS.fetch(test_name.downcase, test_name)
+      end
+
+      def self.structify(object)
+        if object.is_a?(Hash)
+          object.each_with_object(OpenStruct.new) do |kv_pair, struct|
+            key, value = kv_pair
+
+            struct[key] = structify(value)
+          end
+        elsif object.respond_to?(:map)
+          object.map { |item| structify(item) }
+        else
+          object
+        end
+      end
+
+      def self.lab_user
+        user = User.find_by_username('lab_daemon')
+        return user if user
+
+        god_user = User.first
+
+        person = Person.create!(creator: god_user.user_id)
+        PersonName.create!(person: person, given_name: 'Lab', family_name: 'Daemon', creator: god_user.user_id)
+
+        User.create!(username: 'lab_daemon', person: person, creator: god_user.user_id)
+      end
+
+      def self.parse_date(str_date, fallback_date = nil)
+        str_date = str_date&.to_s
+
+        if str_date.blank? && fallback_date.blank?
+          raise "Can't parse blank date"
+        end
+
+        return parse_date(fallback_date) if str_date.blank?
+
+        str_date = str_date.gsub(/^00/, '20').gsub(/^180/, '20')
+
+        case str_date
+        when /\d{4}-\d{2}-\d{2}/
+          str_date
+        when /\d{2}-\d{2}-\d{2}/
+          Date.strptime(str_date, '%d-%m-%Y').strftime('%Y-%m-%d')
+        when /(\d{4}\d{2}\d{2})\d+/
+          Date.strptime(str_date, '%Y%m%d').strftime('%Y-%m-%d')
+        when %r{\d{2}/\d{2}/\d{4}}
+          str_date.to_date.to_s
+        else
+          Rails.logger.warn("Invalid date: #{str_date}")
+          parse_date(fallback_date)
+        end
+      end
+
+      def self.find_concept_by_name(name)
+        ConceptName.joins(:concept)
+                   .merge(Concept.all) # Filter out voided
+                   .where(name: CGI.unescapeHTML(name))
+                   .first
+      end
+    end
+  end
+end

data/app/services/lab/lims/worker.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'logger_multiplexor'
+
+require_relative './api/couchdb_api'
+
+module Lab
+  module Lims
+    ##
+    # Pull/Push orders from/to the LIMS queue (Oops meant CouchDB).
+    module Worker
+      def self.start
+        User.current = Utils.lab_user
+
+        fork(&method(:start_push_worker))
+        fork(&method(:start_pull_worker))
+        fork(&method(:start_realtime_pull_worker)) if realtime_updates_enabled?
+
+        Process.waitall
+      end
+
+      def self.start_push_worker
+        start_worker('push_worker') do
+          worker = PushWorker.new(lims_api)
+
+          worker.push_orders # (wait: true)
+        end
+      end
+
+      def self.start_pull_worker
+        start_worker('pull_worker') do
+          worker = PullWorker.new(lims_api)
+
+          worker.pull_orders
+        end
+      end
+
+      def self.start_realtime_pull_worker
+        start_worker('realtime_pull_worker') do
+          worker = PullWorker.new(Lims::Api::WsApi.new(Lab::Lims::Config.updates_socket))
+
+          worker.pull_orders
+        end
+      end
+
+      LOG_FILES_TO_KEEP = 5
+      LOG_FILE_SIZE = 500.megabytes
+
+      def self.start_worker(worker_name)
+        Rails.logger = LoggerMultiplexor.new(file_logger(worker_name), $stdout)
+        ActiveRecord::Base.logger = Rails.logger
+        Rails.logger.level = :debug
+
+        File.open(log_path("#{worker_name}.lock"), File::RDWR | File::CREAT, 0o644) do |fout|
+          unless fout.flock(File::LOCK_EX | File::LOCK_NB)
+            Rails.logger.warn("Another process already holds lock #{worker_name} (#{fout.read}), exiting...")
+            break
+          end
+
+          fout.write("Locked by process ##{Process.pid} under process group ##{Process.ppid} at #{Time.now}")
+          fout.flush
+          yield
+        end
+      end
+
+      def self.file_logger(worker_name)
+        Logger.new(log_path("#{worker_name}.log"), LOG_FILES_TO_KEEP, LOG_FILE_SIZE)
+      end
+
+      def self.log_path(filename)
+        Lab::Lims::Utils::LIMS_LOG_PATH.join(filename)
+      end
+
+      def self.realtime_updates_enabled?
+        Lims::Config.updates_socket.key?('url')
+      rescue Lab::Lims::Config::ConfigNotFound => e
+        Rails.logger.warn("Check for realtime updates failed: #{e.message}")
+        false
+      end
+
+      def self.lims_api
+        Lab::Lims::ApiFactory.create_api
+      end
+    end
+  end
+end

data/app/services/lab/metadata.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Lab
+  module Metadata
+    # Concepts
+    REASON_FOR_TEST_CONCEPT_NAME = 'Reason for test'
+    REQUESTING_CLINICIAN_CONCEPT_NAME = 'Person making request'
+    SPECIMEN_TYPE_CONCEPT_NAME = 'Specimen type'
+    TARGET_LAB_CONCEPT_NAME = 'Lab'
+    TEST_RESULT_CONCEPT_NAME = 'Lab test result'
+    TEST_RESULT_INDICATOR_CONCEPT_NAME = 'Lab test result indicator'
+    TEST_TYPE_CONCEPT_NAME = 'Test type'
+    UNKNOWN_SPECIMEN = 'Unknown'
+
+    # Encounter
+    ENCOUNTER_TYPE_NAME = 'Lab'
+
+    # Order types
+    ORDER_TYPE_NAME = 'Lab'
+
+    # Programs
+    LAB_PROGRAM_NAME = 'Laboratory Program'
+  end
+end