his_emr_api_lab 1.1.21 → 1.1.22

data/app/services/lab/lims/migrator.rb

@@ -1,216 +0,0 @@
-# frozen_string_literal: true
-
-require 'csv'
-require 'parallel'
-
-require 'couch_bum/couch_bum'
-require 'logger_multiplexor'
-
-require 'concept'
-require 'concept_name'
-require 'drug_order'
-require 'encounter'
-require 'encounter_type'
-require 'observation'
-require 'order'
-require 'order_type'
-require 'patient'
-require 'patient_identifier'
-require 'patient_identifier_type'
-require 'person'
-require 'person_name'
-require 'program'
-require 'user'
-
-require 'lab/lab_encounter'
-require 'lab/lab_order'
-require 'lab/lab_result'
-require 'lab/lab_test'
-require 'lab/lims_order_mapping'
-require 'lab/lims_failed_import'
-
-require_relative './api/couchdb_api'
-require_relative './config'
-require_relative './pull_worker'
-require_relative './utils'
-
-require_relative '../orders_service'
-require_relative '../results_service'
-require_relative '../tests_service'
-require_relative '../../../serializers/lab/lab_order_serializer'
-require_relative '../../../serializers/lab/result_serializer'
-require_relative '../../../serializers/lab/test_serializer'
-
-require_relative 'order_dto'
-require_relative 'utils'
-
-module Lab
-  module Lims
-    ##
-    # Tools for performing a bulk import of data from LIMS' databases to local OpenMRS database.
-    #
-    # Migration sources supported:
-    #   - MySQL
-    #   - CouchDB
-    #
-    # The sources above can be changed by setting the environment various MIGRATION_SOURCE to
-    # either mysql or couchdb.
-    module Migrator
-      MAX_THREADS = ENV.fetch('MIGRATION_WORKERS', 6).to_i
-
-      ##
-      # A Lab::Lims::Api object that supports crawling of a LIMS CouchDB instance.
-      class CouchDbMigratorApi < Lab::Lims::Api::CouchDbApi
-        def initialize(*args, processes: 1, on_merge_processes: nil, **kwargs)
-          super(*args, **kwargs)
-
-          @processes = processes
-          @on_merge_processes = on_merge_processes
-        end
-
-        def consume_orders(from: nil, **_kwargs)
-          limit = 25_000
-
-          loop do
-            on_merge_processes = ->(_item, index, _result) { @on_merge_processes&.call(from + index) }
-            processes = @processes > 1 ? @processes : 0
-
-            orders = read_orders(from, limit)
-            break if orders.empty?
-
-            Parallel.each(orders, in_processes: processes, finish: on_merge_processes) do |row|
-              next unless row['doc']['type']&.casecmp?('Order')
-
-              User.current = Utils.lab_user
-              yield OrderDTO.new(row['doc']), OpenStruct.new(last_seq: (from || 0) + limit, current_seq: from)
-            end
-
-            from += orders.size
-          end
-        end
-
-        private
-
-        def read_orders(from, batch_size)
-          start_key_param = from ? "&skip=#{from}" : ''
-          url = "_all_docs?include_docs=true&limit=#{batch_size}#{start_key_param}"
-
-          Rails.logger.debug("#{CouchDbMigratorApi}: Pulling orders from LIMS CouchDB: #{url}")
-          response = bum.couch_rest :get, url
-
-          response['rows']
-        end
-      end
-
-      ##
-      # Extends the PullWorker to provide pause/resume capabilities.
-      #
-      # Migrations can be take a long time to complete, in cases where something
-      # went wrong you wouldn't to start all over. This worker thus saves
-      # progress and allows for the process to continue from whether it stopped.
-      class MigrationWorker < PullWorker
-        LOG_FILE_PATH = Utils::LIMS_LOG_PATH.join('migration-last-id.dat')
-
-        attr_reader :rejections
-
-        def initialize(api_class)
-          api = api_class.new(processes: MAX_THREADS, on_merge_processes: method(:save_seq))
-          super(api)
-        end
-
-        def last_seq
-          return 0 unless File.exist?(LOG_FILE_PATH)
-
-          File.open(LOG_FILE_PATH, File::RDONLY) do |file|
-            last_seq = file.read&.strip
-            return last_seq.blank? ? nil : last_seq&.to_i
-          end
-        end
-
-        private
-
-        def save_seq(last_seq)
-          File.open(LOG_FILE_PATH, File::WRONLY | File::CREAT, 0o644) do |file|
-            Rails.logger.debug("Process ##{Parallel.worker_number}: Saving last seq: #{last_seq}")
-            file.flock(File::LOCK_EX)
-            file.write(last_seq.to_s)
-            file.flush
-          end
-        end
-
-        def order_rejected(order_dto, reason)
-          @rejections ||= []
-
-          @rejections << OpenStruct.new(order: order_dto, reason: reason)
-        end
-      end
-
-      def self.save_csv(filename, rows:, headers: nil)
-        CSV.open(filename, File::WRONLY | File::CREAT) do |csv|
-          csv << headers if headers
-          rows.each { |row| csv << row }
-        end
-      end
-
-      # NOTE: LIMS_LOG_PATH below is defined in worker.rb
-      MIGRATION_REJECTIONS_CSV_PATH = Utils::LIMS_LOG_PATH.join('migration-rejections.csv')
-
-      def self.export_rejections(rejections)
-        headers = ['doc_id', 'Accession number', 'NHID', 'First name', 'Last name', 'Reason']
-        rows = (rejections || []).map do |rejection|
-          [
-            rejection.order[:_id],
-            rejection.order[:tracking_number],
-            rejection.order[:patient][:id],
-            rejection.order[:patient][:first_name],
-            rejection.order[:patient][:last_name],
-            rejection.reason
-          ]
-        end
-
-        save_csv(MIGRATION_REJECTIONS_CSV_PATH, headers: headers, rows: rows)
-      end
-
-      MIGRATION_FAILURES_CSV_PATH = Utils::LIMS_LOG_PATH.join('migration-failures.csv')
-
-      def self.export_failures
-        headers = ['doc_id', 'Accession number', 'NHID', 'Reason', 'Difference']
-        rows = Lab::LimsFailedImport.all.map do |failure|
-          [
-            failure.lims_id,
-            failure.tracking_number,
-            failure.patient_nhid,
-            failure.reason,
-            failure.diff
-          ]
-        end
-
-        save_csv(MIGRATION_FAILURES_CSV_PATH, headers: headers, rows: rows)
-      end
-
-      MIGRATION_LOG_PATH = Utils::LIMS_LOG_PATH.join('migration.log')
-
-      def self.start_migration
-        Dir.mkdir(Utils::LIMS_LOG_PATH) unless File.exist?(Utils::LIMS_LOG_PATH)
-
-        logger = LoggerMultiplexor.new(Logger.new($stdout), MIGRATION_LOG_PATH)
-        logger.level = :debug
-        Rails.logger = logger
-        ActiveRecord::Base.logger = logger
-        # CouchBum.logger = logger
-
-        api_class = case ENV.fetch('MIGRATION_SOURCE', 'couchdb').downcase
-                    when 'couchdb' then CouchDbMigratorApi
-                    when 'mysql' then Api::MysqlApi
-                    else raise "Invalid MIGRATION_SOURCE: #{ENV['MIGRATION_SOURCE']}"
-                    end
-
-        worker = MigrationWorker.new(api_class)
-        worker.pull_orders(batch_size: 10_000)
-      ensure
-        worker && export_rejections(worker.rejections)
-        export_failures
-      end
-    end
-  end
-end

data/app/services/lab/lims/order_dto.rb

@@ -1,105 +0,0 @@
-# frozen_string_literal: true
-
-require_relative './exceptions'
-
-module Lab
-  module Lims
-    ##
-    # LIMS' Data Transfer Object for orders
-    class OrderDTO < ActiveSupport::HashWithIndifferentAccess
-      include Utils
-
-      ##
-      # Unpacks a LIMS order into an object that OrdersService can handle
-      def to_order_service_params(patient_id:)
-        ActiveSupport::HashWithIndifferentAccess.new(
-          program_id: lab_program.program_id,
-          accession_number: self['tracking_number'],
-          patient_id: patient_id,
-          specimen: { concept_id: specimen_type_id },
-          tests: self['tests']&.map { |test| { concept_id: test_type_id(test) } },
-          requesting_clinician: requesting_clinician,
-          date: start_date,
-          target_lab: facility_name(self['receiving_facility']),
-          order_location: facility_name(self['sending_facility']),
-          reason_for_test_id: reason_for_test
-        )
-      end
-
-      private
-
-      # Translates a LIMS specimen name to an OpenMRS concept_id
-      def specimen_type_id
-        lims_specimen_name = self['sample_type']&.strip&.downcase
-
-        if lims_specimen_name.nil? || %w[specimen_not_collected not_assigned not_specified].include?(lims_specimen_name)
-          return ConceptName.select(:concept_id).find_by_name!('Unknown').concept_id
-        end
-
-        concept = Utils.find_concept_by_name(lims_specimen_name)
-        return concept.concept_id if concept
-
-        raise UnknownSpecimenType, "Unknown specimen name: #{lims_specimen_name}"
-      end
-
-      # Translates a LIMS test type name to an OpenMRS concept_id
-      def test_type_id(lims_test_name)
-        lims_test_name = Utils.translate_test_name(lims_test_name)
-        concept = Utils.find_concept_by_name(lims_test_name)
-        return concept.concept_id if concept
-
-        raise UnknownTestType, "Unknown test type: #{lims_test_name}"
-      end
-
-      # Extract requesting clinician name from LIMS
-      def requesting_clinician
-        return 'Unknown' unless self['who_order_test']
-
-        # TODO: Extend requesting clinician to an obs tree having extra parameters
-        # like phone number and ID to closely match the lims user.
-        first_name = self['who_order_test']['first_name'] || ''
-        last_name = self['who_order_test']['last_name'] || ''
-
-        if first_name.blank? && last_name.blank?
-          logger.warn('Missing requesting clinician name')
-          return ''
-        end
-
-        "#{first_name} #{last_name}"
-      end
-
-      def start_date
-        raise LimsException, 'Order missing created date' if self['date_created'].blank?
-
-        Utils.parse_date(self['date_created'])
-      end
-
-      # Parses a LIMS facility name
-      def facility_name(lims_target_lab)
-        return 'Unknown' if lims_target_lab == 'not_assigned'
-
-        lims_target_lab
-      end
-
-      # Translates a LIMS sample priority to a concept_id
-      def reason_for_test
-        return nil unless self['priority']
-
-        name = case self['priority']
-               when %r{Reapet / Missing}i then 'Repeat / Missing'
-               else self['priority']
-               end
-
-        ConceptName.find_by_name!(name).concept_id
-      end
-
-      def lab_program
-        Program.find_by_name!(Lab::Metadata::LAB_PROGRAM_NAME)
-      end
-
-      def unknown_concept
-        ConceptName.find_by_name!('Unknown')
-      end
-    end
-  end
-end

data/app/services/lab/lims/order_serializer.rb

@@ -1,244 +0,0 @@
-# frozen_string_literal: true
-
-require_relative './config'
-require_relative './order_dto'
-require_relative './utils'
-
-module Lab
-  module Lims
-    ##
-    # Serializes a LabOrder into a LIMS OrderDTO.
-    module OrderSerializer
-      class << self
-        include Utils
-
-        def serialize_order(order)
-          serialized_order = Lims::Utils.structify(Lab::LabOrderSerializer.serialize_order(order))
-
-          Lims::OrderDTO.new(
-            _id: Lab::LimsOrderMapping.find_by(order: order)&.lims_id || serialized_order.accession_number,
-            tracking_number: serialized_order.accession_number,
-            sending_facility: current_facility_name,
-            receiving_facility: serialized_order.target_lab,
-            tests: serialized_order.tests.map { |test| format_test_name(test.name) },
-            patient: format_patient(serialized_order.patient_id),
-            order_location: format_order_location(serialized_order.encounter_id),
-            sample_type: format_sample_type(serialized_order.specimen.name),
-            sample_status: format_sample_status(serialized_order.specimen.name),
-            sample_statuses: format_sample_status_trail(order),
-            test_statuses: format_test_status_trail(order),
-            who_order_test: format_orderer(order),
-            districy: current_district, # yes districy [sic]...
-            priority: format_sample_priority(serialized_order.reason_for_test.name),
-            date_created: serialized_order.order_date,
-            test_results: format_test_results(serialized_order),
-            type: 'Order'
-          )
-        end
-
-        private
-
-        def format_order_location(encounter_id)
-          location_id = Encounter.select(:location_id).where(encounter_id: encounter_id)
-          location = Location.select(:name)
-                             .where(location_id: location_id)
-                             .first
-
-          location&.name
-        end
-
-        # Format patient into a structure that LIMS expects
-        def format_patient(patient_id)
-          person = Person.find(patient_id)
-          name = PersonName.find_by_person_id(patient_id)
-          national_id = PatientIdentifier.joins(:type)
-                                         .merge(PatientIdentifierType.where(name: 'National ID'))
-                                         .where(patient_id: patient_id)
-                                         .first
-          phone_number = PersonAttribute.joins(:type)
-                                        .merge(PersonAttributeType.where(name: 'Cell phone Number'))
-                                        .where(person_id: patient_id)
-                                        .first
-
-          {
-            first_name: name&.given_name,
-            last_name: name&.family_name,
-            id: national_id&.identifier,
-            arv_number: find_arv_number(patient_id),
-            art_regimen: find_current_regimen(patient_id),
-            art_start_date: find_art_start_date(patient_id),
-            phone_number: phone_number&.value || 'Unknown',
-            gender: person.gender,
-            email: nil
-          }
-        end
-
-        def find_current_regimen(patient_id)
-          regimen_data = ActiveRecord::Base.connection.select_one <<~SQL
-            SELECT patient_current_regimen(#{patient_id}, current_date()) regimen
-          SQL
-          return nil if regimen_data.blank?
-
-          regimen_data['regimen']
-        end
-
-        def find_arv_number(patient_id)
-          PatientIdentifier.joins(:type)
-                           .merge(PatientIdentifierType.where(name: 'ARV Number'))
-                           .where(patient_id: patient_id)
-                           .first&.identifier
-        end
-
-        def find_art_start_date(patient_id)
-          start_date = ActiveRecord::Base.connection.select_one <<~SQL
-            SELECT date_antiretrovirals_started(#{patient_id}, current_date()) AS earliest_date
-          SQL
-          return nil if start_date.blank?
-
-          start_date['earliest_date']
-        end
-
-        def format_sample_type(name)
-          return 'not_specified' if name.casecmp?('Unknown')
-
-          return 'CSF' if name.casecmp?('Cerebrospinal Fluid')
-
-          name.titleize
-        end
-
-        def format_sample_status(name)
-          name.casecmp?('Unknown') ? 'specimen_not_collected' : 'specimen_collected'
-        end
-
-        def format_sample_status_trail(order)
-          return [] if order.concept_id == ConceptName.find_by_name!('Unknown').concept_id
-
-          user = User.find(order.discontinued_by || order.creator)
-          drawn_by = PersonName.find_by_person_id(user.user_id)
-          drawn_date = order.discontinued_date || order.start_date
-
-          [
-            drawn_date.strftime('%Y%m%d%H%M%S') => {
-              'status' => 'Drawn',
-              'updated_by' => {
-                'first_name' => drawn_by&.given_name || user.username,
-                'last_name' => drawn_by&.family_name,
-                'phone_number' => nil,
-                'id' => user.username
-              }
-            }
-          ]
-        end
-
-        def format_test_status_trail(order)
-          tests = order.voided.zero? ? order.tests : Lab::LabOrderSerializer.voided_tests(order)
-
-          tests.each_with_object({}) do |test, trail|
-            test_name = format_test_name(ConceptName.find_by_concept_id!(test.value_coded).name)
-
-            current_test_trail = trail[test_name] = {}
-
-            current_test_trail[test.obs_datetime.strftime('%Y%m%d%H%M%S')] = {
-              status: 'Drawn',
-              updated_by: find_user(test.creator)
-            }
-
-            unless test.voided.zero?
-              current_test_trail[test.date_voided.strftime('%Y%m%d%H%M%S')] = {
-                status: 'Voided',
-                updated_by: find_user(test.voided_by)
-              }
-            end
-
-            next unless test.result
-
-            current_test_trail[test.obs_datetime.strftime('%Y%m%d%H%M%S')] = {
-              status: 'Verified',
-              updated_by: find_user(test.result.creator)
-            }
-          end
-        end
-
-        def format_orderer(order)
-          find_user(order.creator)
-        end
-
-        def format_test_results(order)
-          order.tests&.each_with_object({}) do |test, results|
-            next if test.result.nil? || test.result.empty?
-
-            test_creator = User.find(Observation.find(test.result.first.id).creator)
-            test_creator_name = PersonName.find_by_person_id(test_creator.person_id)
-
-            results[format_test_name(test.name)] = {
-              results: test.result.each_with_object({}) do |measure, measures|
-                measures[format_test_name(measure.indicator.name)] = {
-                  result_value: "#{measure.value_modifier}#{measure.value}"
-                }
-              end,
-              result_date: test.result.first&.date,
-              result_entered_by: {
-                first_name: test_creator_name&.given_name,
-                last_name: test_creator_name&.family_name,
-                id: test_creator.username
-              }
-            }
-          end
-        end
-
-        def format_test_name(test_name)
-          return 'Viral Load' if test_name.casecmp?('HIV Viral load')
-
-          return 'TB' if test_name.casecmp?('TB Program')
-
-          test_name.titleize
-        end
-
-        def format_sample_priority(priority)
-          return 'Routine' if priority&.casecmp?('Medical examination, routine')
-
-          priority&.titleize
-        end
-
-        def current_health_center
-          health_center = Location.current_health_center
-          raise 'Current health center not set' unless health_center
-
-          health_center
-        end
-
-        def current_district
-          district = current_health_center.city_village\
-                       || current_health_center.parent&.name\
-                       || GlobalProperty.find_by_property('current_health_center_district')&.property_value
-
-          return district if district
-
-          GlobalProperty.create(property: 'current_health_center_district',
-                                property_value: Lims::Config.application['district'],
-                                uuid: SecureRandom.uuid)
-
-          Config.application['district']
-        end
-
-        def current_facility_name
-          current_health_center.name
-        end
-
-        def find_user(user_id)
-          user = User.find(user_id)
-          person_name = PersonName.find_by(person_id: user.person_id)
-          phone_number = PersonAttribute.find_by(type: PersonAttributeType.where(name: 'Cell phone number'),
-                                                 person_id: user.person_id)
-
-          {
-            first_name: person_name&.given_name,
-            last_name: person_name&.family_name,
-            phone_number: phone_number&.value,
-            id: user.username
-          }
-        end
-      end
-    end
-  end
-end