his_emr_api_lab 1.1.22 → 1.1.23

data/app/services/lab/lims/migrator.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'parallel'
+
+require 'couch_bum/couch_bum'
+require 'logger_multiplexor'
+
+require 'concept'
+require 'concept_name'
+require 'drug_order'
+require 'encounter'
+require 'encounter_type'
+require 'observation'
+require 'order'
+require 'order_type'
+require 'patient'
+require 'patient_identifier'
+require 'patient_identifier_type'
+require 'person'
+require 'person_name'
+require 'program'
+require 'user'
+
+require 'lab/lab_encounter'
+require 'lab/lab_order'
+require 'lab/lab_result'
+require 'lab/lab_test'
+require 'lab/lims_order_mapping'
+require 'lab/lims_failed_import'
+
+require_relative './api/couchdb_api'
+require_relative './config'
+require_relative './pull_worker'
+require_relative './utils'
+
+require_relative '../orders_service'
+require_relative '../results_service'
+require_relative '../tests_service'
+require_relative '../../../serializers/lab/lab_order_serializer'
+require_relative '../../../serializers/lab/result_serializer'
+require_relative '../../../serializers/lab/test_serializer'
+
+require_relative 'order_dto'
+require_relative 'utils'
+
+module Lab
+  module Lims
+    ##
+    # Tools for performing a bulk import of data from LIMS' databases to local OpenMRS database.
+    #
+    # Migration sources supported:
+    #   - MySQL
+    #   - CouchDB
+    #
+    # The sources above can be changed by setting the environment various MIGRATION_SOURCE to
+    # either mysql or couchdb.
+    module Migrator
+      MAX_THREADS = ENV.fetch('MIGRATION_WORKERS', 6).to_i
+
+      ##
+      # A Lab::Lims::Api object that supports crawling of a LIMS CouchDB instance.
+      class CouchDbMigratorApi < Lab::Lims::Api::CouchDbApi
+        def initialize(*args, processes: 1, on_merge_processes: nil, **kwargs)
+          super(*args, **kwargs)
+
+          @processes = processes
+          @on_merge_processes = on_merge_processes
+        end
+
+        def consume_orders(from: nil, **_kwargs)
+          limit = 25_000
+
+          loop do
+            on_merge_processes = ->(_item, index, _result) { @on_merge_processes&.call(from + index) }
+            processes = @processes > 1 ? @processes : 0
+
+            orders = read_orders(from, limit)
+            break if orders.empty?
+
+            Parallel.each(orders, in_processes: processes, finish: on_merge_processes) do |row|
+              next unless row['doc']['type']&.casecmp?('Order')
+
+              User.current = Utils.lab_user
+              yield OrderDTO.new(row['doc']), OpenStruct.new(last_seq: (from || 0) + limit, current_seq: from)
+            end
+
+            from += orders.size
+          end
+        end
+
+        private
+
+        def read_orders(from, batch_size)
+          start_key_param = from ? "&skip=#{from}" : ''
+          url = "_all_docs?include_docs=true&limit=#{batch_size}#{start_key_param}"
+
+          Rails.logger.debug("#{CouchDbMigratorApi}: Pulling orders from LIMS CouchDB: #{url}")
+          response = bum.couch_rest :get, url
+
+          response['rows']
+        end
+      end
+
+      ##
+      # Extends the PullWorker to provide pause/resume capabilities.
+      #
+      # Migrations can be take a long time to complete, in cases where something
+      # went wrong you wouldn't to start all over. This worker thus saves
+      # progress and allows for the process to continue from whether it stopped.
+      class MigrationWorker < PullWorker
+        LOG_FILE_PATH = Utils::LIMS_LOG_PATH.join('migration-last-id.dat')
+
+        attr_reader :rejections
+
+        def initialize(api_class)
+          api = api_class.new(processes: MAX_THREADS, on_merge_processes: method(:save_seq))
+          super(api)
+        end
+
+        def last_seq
+          return 0 unless File.exist?(LOG_FILE_PATH)
+
+          File.open(LOG_FILE_PATH, File::RDONLY) do |file|
+            last_seq = file.read&.strip
+            return last_seq.blank? ? nil : last_seq&.to_i
+          end
+        end
+
+        private
+
+        def save_seq(last_seq)
+          File.open(LOG_FILE_PATH, File::WRONLY | File::CREAT, 0o644) do |file|
+            Rails.logger.debug("Process ##{Parallel.worker_number}: Saving last seq: #{last_seq}")
+            file.flock(File::LOCK_EX)
+            file.write(last_seq.to_s)
+            file.flush
+          end
+        end
+
+        def order_rejected(order_dto, reason)
+          @rejections ||= []
+
+          @rejections << OpenStruct.new(order: order_dto, reason: reason)
+        end
+      end
+
+      def self.save_csv(filename, rows:, headers: nil)
+        CSV.open(filename, File::WRONLY | File::CREAT) do |csv|
+          csv << headers if headers
+          rows.each { |row| csv << row }
+        end
+      end
+
+      # NOTE: LIMS_LOG_PATH below is defined in worker.rb
+      MIGRATION_REJECTIONS_CSV_PATH = Utils::LIMS_LOG_PATH.join('migration-rejections.csv')
+
+      def self.export_rejections(rejections)
+        headers = ['doc_id', 'Accession number', 'NHID', 'First name', 'Last name', 'Reason']
+        rows = (rejections || []).map do |rejection|
+          [
+            rejection.order[:_id],
+            rejection.order[:tracking_number],
+            rejection.order[:patient][:id],
+            rejection.order[:patient][:first_name],
+            rejection.order[:patient][:last_name],
+            rejection.reason
+          ]
+        end
+
+        save_csv(MIGRATION_REJECTIONS_CSV_PATH, headers: headers, rows: rows)
+      end
+
+      MIGRATION_FAILURES_CSV_PATH = Utils::LIMS_LOG_PATH.join('migration-failures.csv')
+
+      def self.export_failures
+        headers = ['doc_id', 'Accession number', 'NHID', 'Reason', 'Difference']
+        rows = Lab::LimsFailedImport.all.map do |failure|
+          [
+            failure.lims_id,
+            failure.tracking_number,
+            failure.patient_nhid,
+            failure.reason,
+            failure.diff
+          ]
+        end
+
+        save_csv(MIGRATION_FAILURES_CSV_PATH, headers: headers, rows: rows)
+      end
+
+      MIGRATION_LOG_PATH = Utils::LIMS_LOG_PATH.join('migration.log')
+
+      def self.start_migration
+        Dir.mkdir(Utils::LIMS_LOG_PATH) unless File.exist?(Utils::LIMS_LOG_PATH)
+
+        logger = LoggerMultiplexor.new(Logger.new($stdout), MIGRATION_LOG_PATH)
+        logger.level = :debug
+        Rails.logger = logger
+        ActiveRecord::Base.logger = logger
+        # CouchBum.logger = logger
+
+        api_class = case ENV.fetch('MIGRATION_SOURCE', 'couchdb').downcase
+                    when 'couchdb' then CouchDbMigratorApi
+                    when 'mysql' then Api::MysqlApi
+                    else raise "Invalid MIGRATION_SOURCE: #{ENV['MIGRATION_SOURCE']}"
+                    end
+
+        worker = MigrationWorker.new(api_class)
+        worker.pull_orders(batch_size: 10_000)
+      ensure
+        worker && export_rejections(worker.rejections)
+        export_failures
+      end
+    end
+  end
+end

data/app/services/lab/lims/order_dto.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require_relative './exceptions'
+
+module Lab
+  module Lims
+    ##
+    # LIMS' Data Transfer Object for orders
+    class OrderDTO < ActiveSupport::HashWithIndifferentAccess
+      include Utils
+
+      ##
+      # Unpacks a LIMS order into an object that OrdersService can handle
+      def to_order_service_params(patient_id:)
+        ActiveSupport::HashWithIndifferentAccess.new(
+          program_id: lab_program.program_id,
+          accession_number: self['tracking_number'],
+          patient_id: patient_id,
+          specimen: { concept_id: specimen_type_id },
+          tests: self['tests']&.map { |test| { concept_id: test_type_id(test) } },
+          requesting_clinician: requesting_clinician,
+          date: start_date,
+          target_lab: facility_name(self['receiving_facility']),
+          order_location: facility_name(self['sending_facility']),
+          reason_for_test_id: reason_for_test
+        )
+      end
+
+      private
+
+      # Translates a LIMS specimen name to an OpenMRS concept_id
+      def specimen_type_id
+        lims_specimen_name = self['sample_type']&.strip&.downcase
+
+        if lims_specimen_name.nil? || %w[specimen_not_collected not_assigned not_specified].include?(lims_specimen_name)
+          return ConceptName.select(:concept_id).find_by_name!('Unknown').concept_id
+        end
+
+        concept = Utils.find_concept_by_name(lims_specimen_name)
+        return concept.concept_id if concept
+
+        raise UnknownSpecimenType, "Unknown specimen name: #{lims_specimen_name}"
+      end
+
+      # Translates a LIMS test type name to an OpenMRS concept_id
+      def test_type_id(lims_test_name)
+        lims_test_name = Utils.translate_test_name(lims_test_name)
+        concept = Utils.find_concept_by_name(lims_test_name)
+        return concept.concept_id if concept
+
+        raise UnknownTestType, "Unknown test type: #{lims_test_name}"
+      end
+
+      # Extract requesting clinician name from LIMS
+      def requesting_clinician
+        return 'Unknown' unless self['who_order_test']
+
+        # TODO: Extend requesting clinician to an obs tree having extra parameters
+        # like phone number and ID to closely match the lims user.
+        first_name = self['who_order_test']['first_name'] || ''
+        last_name = self['who_order_test']['last_name'] || ''
+
+        if first_name.blank? && last_name.blank?
+          logger.warn('Missing requesting clinician name')
+          return ''
+        end
+
+        "#{first_name} #{last_name}"
+      end
+
+      def start_date
+        raise LimsException, 'Order missing created date' if self['date_created'].blank?
+
+        Utils.parse_date(self['date_created'])
+      end
+
+      # Parses a LIMS facility name
+      def facility_name(lims_target_lab)
+        return 'Unknown' if lims_target_lab == 'not_assigned'
+
+        lims_target_lab
+      end
+
+      # Translates a LIMS sample priority to a concept_id
+      def reason_for_test
+        return nil unless self['priority']
+
+        name = case self['priority']
+               when %r{Reapet / Missing}i then 'Repeat / Missing'
+               else self['priority']
+               end
+
+        ConceptName.find_by_name!(name).concept_id
+      end
+
+      def lab_program
+        Program.find_by_name!(Lab::Metadata::LAB_PROGRAM_NAME)
+      end
+
+      def unknown_concept
+        ConceptName.find_by_name!('Unknown')
+      end
+    end
+  end
+end

data/app/services/lab/lims/order_serializer.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require_relative './config'
+require_relative './order_dto'
+require_relative './utils'
+
+module Lab
+  module Lims
+    ##
+    # Serializes a LabOrder into a LIMS OrderDTO.
+    module OrderSerializer
+      class << self
+        include Utils
+
+        def serialize_order(order)
+          serialized_order = Lims::Utils.structify(Lab::LabOrderSerializer.serialize_order(order))
+
+          Lims::OrderDTO.new(
+            _id: Lab::LimsOrderMapping.find_by(order: order)&.lims_id || serialized_order.accession_number,
+            tracking_number: serialized_order.accession_number,
+            sending_facility: current_facility_name,
+            receiving_facility: serialized_order.target_lab,
+            tests: serialized_order.tests.map { |test| format_test_name(test.name) },
+            patient: format_patient(serialized_order.patient_id),
+            order_location: format_order_location(serialized_order.encounter_id),
+            sample_type: format_sample_type(serialized_order.specimen.name),
+            sample_status: format_sample_status(serialized_order.specimen.name),
+            sample_statuses: format_sample_status_trail(order),
+            test_statuses: format_test_status_trail(order),
+            who_order_test: format_orderer(order),
+            districy: current_district, # yes districy [sic]...
+            priority: format_sample_priority(serialized_order.reason_for_test.name),
+            date_created: serialized_order.order_date,
+            test_results: format_test_results(serialized_order),
+            type: 'Order'
+          )
+        end
+
+        private
+
+        def format_order_location(encounter_id)
+          location_id = Encounter.select(:location_id).where(encounter_id: encounter_id)
+          location = Location.select(:name)
+                             .where(location_id: location_id)
+                             .first
+
+          location&.name
+        end
+
+        # Format patient into a structure that LIMS expects
+        def format_patient(patient_id)
+          person = Person.find(patient_id)
+          name = PersonName.find_by_person_id(patient_id)
+          national_id = PatientIdentifier.joins(:type)
+                                         .merge(PatientIdentifierType.where(name: 'National ID'))
+                                         .where(patient_id: patient_id)
+                                         .first
+          phone_number = PersonAttribute.joins(:type)
+                                        .merge(PersonAttributeType.where(name: 'Cell phone Number'))
+                                        .where(person_id: patient_id)
+                                        .first
+
+          {
+            first_name: name&.given_name,
+            last_name: name&.family_name,
+            id: national_id&.identifier,
+            arv_number: find_arv_number(patient_id),
+            art_regimen: find_current_regimen(patient_id),
+            art_start_date: find_art_start_date(patient_id),
+            phone_number: phone_number&.value || 'Unknown',
+            gender: person.gender,
+            email: nil
+          }
+        end
+
+        def find_current_regimen(patient_id)
+          regimen_data = ActiveRecord::Base.connection.select_one <<~SQL
+            SELECT patient_current_regimen(#{patient_id}, current_date()) regimen
+          SQL
+          return nil if regimen_data.blank?
+
+          regimen_data['regimen']
+        end
+
+        def find_arv_number(patient_id)
+          PatientIdentifier.joins(:type)
+                           .merge(PatientIdentifierType.where(name: 'ARV Number'))
+                           .where(patient_id: patient_id)
+                           .first&.identifier
+        end
+
+        def find_art_start_date(patient_id)
+          start_date = ActiveRecord::Base.connection.select_one <<~SQL
+            SELECT date_antiretrovirals_started(#{patient_id}, current_date()) AS earliest_date
+          SQL
+          return nil if start_date.blank?
+
+          start_date['earliest_date']
+        end
+
+        def format_sample_type(name)
+          return 'not_specified' if name.casecmp?('Unknown')
+
+          return 'CSF' if name.casecmp?('Cerebrospinal Fluid')
+
+          name.titleize
+        end
+
+        def format_sample_status(name)
+          name.casecmp?('Unknown') ? 'specimen_not_collected' : 'specimen_collected'
+        end
+
+        def format_sample_status_trail(order)
+          return [] if order.concept_id == ConceptName.find_by_name!('Unknown').concept_id
+
+          user = User.find(order.discontinued_by || order.creator)
+          drawn_by = PersonName.find_by_person_id(user.user_id)
+          drawn_date = order.discontinued_date || order.start_date
+
+          [
+            drawn_date.strftime('%Y%m%d%H%M%S') => {
+              'status' => 'Drawn',
+              'updated_by' => {
+                'first_name' => drawn_by&.given_name || user.username,
+                'last_name' => drawn_by&.family_name,
+                'phone_number' => nil,
+                'id' => user.username
+              }
+            }
+          ]
+        end
+
+        def format_test_status_trail(order)
+          tests = order.voided.zero? ? order.tests : Lab::LabOrderSerializer.voided_tests(order)
+
+          tests.each_with_object({}) do |test, trail|
+            test_name = format_test_name(ConceptName.find_by_concept_id!(test.value_coded).name)
+
+            current_test_trail = trail[test_name] = {}
+
+            current_test_trail[test.obs_datetime.strftime('%Y%m%d%H%M%S')] = {
+              status: 'Drawn',
+              updated_by: find_user(test.creator)
+            }
+
+            unless test.voided.zero?
+              current_test_trail[test.date_voided.strftime('%Y%m%d%H%M%S')] = {
+                status: 'Voided',
+                updated_by: find_user(test.voided_by)
+              }
+            end
+
+            next unless test.result
+
+            current_test_trail[test.obs_datetime.strftime('%Y%m%d%H%M%S')] = {
+              status: 'Verified',
+              updated_by: find_user(test.result.creator)
+            }
+          end
+        end
+
+        def format_orderer(order)
+          find_user(order.creator)
+        end
+
+        def format_test_results(order)
+          order.tests&.each_with_object({}) do |test, results|
+            next if test.result.nil? || test.result.empty?
+
+            test_creator = User.find(Observation.find(test.result.first.id).creator)
+            test_creator_name = PersonName.find_by_person_id(test_creator.person_id)
+
+            results[format_test_name(test.name)] = {
+              results: test.result.each_with_object({}) do |measure, measures|
+                measures[format_test_name(measure.indicator.name)] = {
+                  result_value: "#{measure.value_modifier}#{measure.value}"
+                }
+              end,
+              result_date: test.result.first&.date,
+              result_entered_by: {
+                first_name: test_creator_name&.given_name,
+                last_name: test_creator_name&.family_name,
+                id: test_creator.username
+              }
+            }
+          end
+        end
+
+        def format_test_name(test_name)
+          return 'Viral Load' if test_name.casecmp?('HIV Viral load')
+
+          return 'TB' if test_name.casecmp?('TB Program')
+
+          test_name.titleize
+        end
+
+        def format_sample_priority(priority)
+          return 'Routine' if priority&.casecmp?('Medical examination, routine')
+
+          priority&.titleize
+        end
+
+        def current_health_center
+          health_center = Location.current_health_center
+          raise 'Current health center not set' unless health_center
+
+          health_center
+        end
+
+        def current_district
+          district = current_health_center.city_village\
+                       || current_health_center.parent&.name\
+                       || GlobalProperty.find_by_property('current_health_center_district')&.property_value
+
+          return district if district
+
+          GlobalProperty.create(property: 'current_health_center_district',
+                                property_value: Lims::Config.application['district'],
+                                uuid: SecureRandom.uuid)
+
+          Config.application['district']
+        end
+
+        def current_facility_name
+          current_health_center.name
+        end
+
+        def find_user(user_id)
+          user = User.find(user_id)
+          person_name = PersonName.find_by(person_id: user.person_id)
+          phone_number = PersonAttribute.find_by(type: PersonAttributeType.where(name: 'Cell phone number'),
+                                                 person_id: user.person_id)
+
+          {
+            first_name: person_name&.given_name,
+            last_name: person_name&.family_name,
+            phone_number: phone_number&.value,
+            id: user.username
+          }
+        end
+      end
+    end
+  end
+end