his_emr_api_lab 0.0.11 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 060337303a9a04fc0f9cebbe525f2275964b41776297b440bc9d7ea4c25c39b8
-  data.tar.gz: 5cc5383ab62d941de6a0d5fa2f51c5199461f141459cea072b40c004277855d1
+  metadata.gz: 7e6be868d3a1113b834bf4e6644734eead3e01d7b51f90e310e3020eb8125b6c
+  data.tar.gz: bd7ca74bdf0af442d3a4c6f9a1f9e0a73d4d61571f1e39ae6ce5352246cd801d
 SHA512:
-  metadata.gz: 25e70f8e501dd0f0edf25b6f202f41129842fe5099cd5e6d977035df063d9233ac38b98e51ebdd2c2bcb382524da111b42138815d0a48c02e5bdb979b43c8127
-  data.tar.gz: 950d9d988b026ac2dbaed9f63be620cc474816aaba77c64231291e721e463f2872a20c2f629a553d1c52c0689e20afb28d44fa1c981807e1a0c4ed7945d6103f
+  metadata.gz: 7c122f5fbc967d7e470aac9eb455f77d9471a40caa582a9306d4558a25094d8efba17a9aca25ea312609ac9c7571cf154f463a0c65a55387bb8a030dcf48b7c6
+  data.tar.gz: 9c68ca05bb1accaac6942c37f2399ae5ef5f94bde87cbd30811f35e394a9b2a43e2671f266173839156b980c3030051fc5e9fc4103c3b8f41a7dcd84e5324e35

data/app/services/lab/labelling_service/order_label.rb

@@ -60,7 +60,7 @@ module Lab
       def drawer
         return 'N/A' if order.concept_id == unknown_concept.concept_id
 
-        drawer_id = order.discontinued_by || order.creator
+        drawer_id = User.find(order.discontinued_by || order.creator).person_id
         draw_date = (order.discontinued_date || order.start_date).strftime('%d/%^b/%Y %H:%M:%S')
 
         name = PersonName.find_by_person_id(drawer_id)

data/app/services/lab/lims/api/couchdb_api.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'couch_bum/couch_bum'
+
+require_relative '../config'
+
+module Lab
+  module Lims
+    module Api
+      ##
+      # Talk to LIMS like a boss
+      class CouchDbApi
+        attr_reader :bum
+
+        def initialize(config: nil)
+          config ||= Config.couchdb
+
+          @bum = CouchBum.new(protocol: config['protocol'],
+                              host: config['host'],
+                              port: config['port'],
+                              database: "#{config['prefix']}_order_#{config['suffix']}",
+                              username: config['username'],
+                              password: config['password'])
+        end
+
+        ##
+        # Consume orders from the LIMS queue.
+        #
+        # Retrieves orders from the LIMS queue and passes each order to
+        # given block until the queue is empty or connection is terminated
+        # by calling method +choke+.
+        def consume_orders(from: 0, limit: 30)
+          bum.binge_changes(since: from, limit: limit, include_docs: true) do |change|
+            next unless change['doc']['type']&.casecmp?('Order')
+
+            yield OrderDTO.new(change['doc']), self
+          end
+        end
+
+        def create_order(order)
+          bum.couch_rest :post, '/', order
+        end
+
+        def update_order(id, order)
+          bum.couch_rest :put, "/#{id}", order
+        end
+      end
+    end
+  end
+end

data/app/services/lab/lims/api/mysql_api.rb

@@ -0,0 +1,316 @@
+# frozen_string_literal: true
+
+module Lab
+  module Lims
+    module Api
+      class MysqlApi
+        def self.start
+          instance = MysqlApi.new
+          orders_processed = 0
+          instance.consume_orders(from: 0, limit: 1000) do |order|
+            puts "Order ##{orders_processed}"
+            pp order
+            orders_processed += 1
+            puts
+          end
+        end
+
+        def initialize(processes: 1, on_merge_processes: nil)
+          @processes = processes
+          @on_merge_processes = on_merge_processes
+          @mysql_connection_pool = {}
+        end
+
+        def multiprocessed?
+          @processes > 1
+        end
+
+        def consume_orders(from: nil, limit: 1000)
+          loop do
+            specimens_to_process = specimens(from, limit)
+            break if specimens_to_process.size.zero?
+
+            processes = multiprocessed? ? @processes : 0
+            on_merge_processes = ->(_item, index, _result) { @on_merge_processes&.call(from + index) }
+
+            Parallel.map(specimens_to_process, in_processes: processes, finish: on_merge_processes) do |specimen|
+              User.current ||= Utils.lab_user
+
+              tests = specimen_tests(specimen['specimen_id'])
+              results = tests.each_with_object({}) do |test, object|
+                object[test['test_name']] = test_results(test['test_id'])
+              end
+
+              dto = make_order_dto(
+                specimen: specimen,
+                patient: specimen_patient(specimen['specimen_id']),
+                test_results: results,
+                specimen_status_trail: specimen_status_trail(specimen['specimen_id']),
+                test_status_trail: tests.each_with_object({}) do |test, trails|
+                  trails[test['test_name']] = test_status_trail(test['test_id'])
+                end
+              )
+
+              yield dto, OpenStruct.new(last_seq: from)
+            end
+
+            from += limit
+          end
+        end
+
+        def parallel_map(items, on_merge: nil, &block); end
+
+        private
+
+        def specimens(start_id, limit)
+          query = <<~SQL
+            SELECT specimen.id AS specimen_id,
+                   specimen.couch_id AS doc_id,
+                   specimen_types.name AS specimen_name,
+                   specimen.tracking_number,
+                   specimen.priority,
+                   specimen.target_lab,
+                   specimen.sending_facility,
+                   specimen.drawn_by_id,
+                   specimen.drawn_by_name,
+                   specimen.drawn_by_phone_number,
+                   specimen.ward_id,
+                   specimen_statuses.name AS specimen_status,
+                   specimen.district,
+                   specimen.date_created AS order_date
+            FROM specimen
+            INNER JOIN specimen_types ON specimen_types.id = specimen.specimen_type_id
+            INNER JOIN specimen_statuses ON specimen_statuses.id = specimen.specimen_status_id
+          SQL
+
+          query = "#{query} WHERE specimen.id > #{sql_escape(start_id)}" if start_id
+          query = "#{query} LIMIT #{limit.to_i}"
+
+          Rails.logger.debug(query)
+          query(query)
+        end
+
+        ##
+        # Pull patient associated with given specimen
+        def specimen_patient(specimen_id)
+          results = query <<~SQL
+            SELECT patients.patient_number AS nhid,
+                   patients.name,
+                   patients.gender,
+                   DATE(patients.dob) AS birthdate
+            FROM patients
+            INNER JOIN tests
+              ON tests.patient_id = patients.id
+              AND tests.specimen_id = #{sql_escape(specimen_id)}
+            LIMIT 1
+          SQL
+
+          results.first
+        end
+
+        def specimen_tests(specimen_id)
+          query <<~SQL
+            SELECT tests.id AS test_id,
+                   test_types.name AS test_name,
+                   tests.created_by AS drawn_by_name
+            FROM tests
+            INNER JOIN test_types ON test_types.id = tests.test_type_id
+            WHERE tests.specimen_id = #{sql_escape(specimen_id)}
+          SQL
+        end
+
+        def specimen_status_trail(specimen_id)
+          query <<~SQL
+            SELECT specimen_statuses.name AS status_name,
+                   specimen_status_trails.who_updated_id AS updated_by_id,
+                   specimen_status_trails.who_updated_name AS updated_by_name,
+                   specimen_status_trails.who_updated_phone_number AS updated_by_phone_number,
+                   specimen_status_trails.time_updated AS date
+            FROM specimen_status_trails
+            INNER JOIN specimen_statuses
+              ON specimen_statuses.id = specimen_status_trails.specimen_status_id
+            WHERE specimen_status_trails.specimen_id = #{sql_escape(specimen_id)}
+          SQL
+        end
+
+        def test_status_trail(test_id)
+          query <<~SQL
+            SELECT test_statuses.name AS status_name,
+                   test_status_trails.who_updated_id AS updated_by_id,
+                   test_status_trails.who_updated_name AS updated_by_name,
+                   test_status_trails.who_updated_phone_number AS updated_by_phone_number,
+                   COALESCE(test_status_trails.time_updated, test_status_trails.created_at) AS date
+            FROM test_status_trails
+            INNER JOIN test_statuses
+              ON test_statuses.id = test_status_trails.test_status_id
+            WHERE test_status_trails.test_id = #{sql_escape(test_id)}
+          SQL
+        end
+
+        def test_results(test_id)
+          query <<~SQL
+            SELECT measures.name AS measure_name,
+                   test_results.result,
+                   test_results.time_entered AS date
+            FROM test_results
+            INNER JOIN measures ON measures.id = test_results.measure_id
+            WHERE test_results.test_id = #{sql_escape(test_id)}
+          SQL
+        end
+
+        def make_order_dto(specimen:, patient:, test_status_trail:, specimen_status_trail:, test_results:)
+          drawn_by_first_name, drawn_by_last_name = specimen['drawn_by_name']&.split
+          patient_first_name, patient_last_name = patient['name'].split
+
+          OrderDTO.new(
+            _id: specimen['doc_id'].blank? ? SecureRandom.uuid : specimen['doc_id'],
+            _rev: '0',
+            tracking_number: specimen['tracking_number'],
+            date_created: specimen['order_date'],
+            sample_type: specimen['specimen_name'],
+            tests: test_status_trail.keys,
+            districy: specimen['district'], # districy [sic] - That's how it's named
+            order_location: specimen['ward_id'],
+            sending_facility: specimen['sending_facility'],
+            receiving_facility: specimen['target_lab'],
+            priority: specimen['priority'],
+            patient: {
+              id: patient['nhid'],
+              first_name: patient_first_name,
+              last_name: patient_last_name,
+              gender: patient['gender'],
+              birthdate: patient['birthdate'],
+              email: nil,
+              phone_number: nil
+            },
+            type: 'Order',
+            who_order_test: {
+              first_name: drawn_by_first_name,
+              last_name: drawn_by_last_name,
+              id: specimen['drawn_by_id'],
+              phone_number: specimen['drawn_by_phone_number']
+            },
+            sample_status: specimen['specimen_status'],
+            sample_statuses: specimen_status_trail.each_with_object({}) do |trail_entry, object|
+              first_name, last_name = trail_entry['updated_by_name'].split
+
+              object[format_date(trail_entry['date'])] = {
+                status: trail_entry['status_name'],
+                updated_by: {
+                  first_name: first_name,
+                  last_name: last_name,
+                  phone_number: trail_entry['updated_by_phone_number'],
+                  id: trail_entry['updated_by_id']
+                }
+              }
+            end,
+            test_statuses: test_status_trail.each_with_object({}) do |trail_entry, formatted_trail|
+              test_name, test_statuses = trail_entry
+
+              formatted_trail[test_name] = test_statuses.each_with_object({}) do |test_status, formatted_statuses|
+                updated_by_first_name, updated_by_last_name = test_status['updated_by_name'].split
+
+                formatted_statuses[format_date(test_status['date'])] = {
+                  status: test_status['status_name'],
+                  updated_by: {
+                    first_name: updated_by_first_name,
+                    last_name: updated_by_last_name,
+                    phone_number: test_status['updated_by_phone_number'],
+                    id: test_status['updated_by_id']
+                  }
+                }
+              end
+            end,
+            test_results: test_results.each_with_object({}) do |results_entry, formatted_results|
+              test_name, results = results_entry
+
+              formatted_results[test_name] = format_test_result_for_dto(test_name, specimen, results, test_status_trail)
+            end
+          )
+        end
+
+        def format_test_result_for_dto(test_name, specimen, results, test_status_trail)
+          return {} if results.size.zero?
+
+          result_create_event = test_status_trail[test_name]&.find do |trail_entry|
+            trail_entry['status_name'].casecmp?('drawn')
+          end
+
+          result_creator_first_name, result_creator_last_name = result_create_event&.fetch('updated_by_name')&.split
+          unless result_creator_first_name
+            result_creator_first_name, result_creator_last_name = specimen['drawn_by_name']&.split
+          end
+
+          {
+            results: results.each_with_object({}) do |result, formatted_measures|
+              formatted_measures[result['measure_name']] = {
+                result_value: result['result']
+              }
+            end,
+            date_result_entered: format_date(result_create_event&.fetch('date') || specimen['order_date'], :iso),
+            result_entered_by: {
+              first_name: result_creator_first_name,
+              last_name: result_creator_last_name,
+              phone_number: result_create_event&.fetch('updated_by_phone_number') || specimen['drawn_by_phone_number'],
+              id: result_create_event&.fetch('updated_by_id') || specimen['updated_by_id']
+            }
+          }
+        end
+
+        def mysql
+          return mysql_connection if mysql_connection
+
+          config = lambda do |key|
+            @config ||= Lab::Lims::Config.database
+            @config['default'][key] || @config['development'][key]
+          end
+
+          connection = Mysql2::Client.new(host: config['host'] || 'localhost',
+                                          username: config['username'] || 'root',
+                                          password: config['password'],
+                                          port: config['port'] || '3306',
+                                          database: config['database'],
+                                          reconnect: true)
+
+          self.mysql_connection = connection
+        end
+
+        def pid
+          return -1 if Parallel.worker_number.nil?
+
+          Parallel.worker_number
+        end
+
+        def mysql_connection=(connection)
+          @mysql_connection_pool[pid] = connection
+        end
+
+        def mysql_connection
+          @mysql_connection_pool[pid]
+        end
+
+        def query(sql)
+          Rails.logger.debug("#{MysqlApi}: #{sql}")
+          mysql.query(sql)
+        end
+
+        def sql_escape(value)
+          mysql.escape(value.to_s)
+        end
+
+        ##
+        # Lims has some weird date formatting standards...
+        def format_date(date, format = nil)
+          date = date&.to_time
+
+          case format
+          when :iso
+            date&.strftime('%Y-%m-%d %H:%M:%S')
+          else
+            date&.strftime('%Y%m%d%H%M%S')
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/lab/lims/config.rb

@@ -29,6 +29,12 @@ module Lab
           YAML.load_file(find_config_path('application.yml'))
         end
 
+        ##
+        # Returns LIMS' database.yml configuration file
+        def database
+          YAML.load_file(find_config_path('database.yml'))
+        end
+
         private
 
         ##
@@ -39,10 +45,11 @@ module Lab
           paths = [
             "#{ENV['HOME']}/apps/nlims_controller/config/#{filename}",
             "/var/www/nlims_controller/config/#{filename}",
-            Rails.root.parent.join("nlims_controller/config/#{filename}"),
-            Rails.root.join('config/lims-couch.yml')
+            Rails.root.parent.join("nlims_controller/config/#{filename}")
           ]
 
+          paths = [Rails.root.join('config/lims-couchdb.yml'), *paths] if filename == 'couchdb.yml'
+
           paths.each do |path|
             Rails.logger.debug("Looking for LIMS couchdb config at: #{path}")
             return path if File.exist?(path)

data/app/services/lab/lims/failed_imports.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative './api'
+require_relative './order_dto'
+
+module Lab
+  module Lims
+    ##
+    # Manage LIMS orders that failed to import
+    module FailedImports
+      class << self
+        ##
+        # Retrieve all imports that failed
+        def failed_imports(start_id = 0, limit = 20)
+          Lab::LimsFailedImport.where('id >= ?', start_id).limit(limit)
+        end
+
+        ##
+        # Forcefully imports a failed import into a patient
+        def force_import(failed_import_id, _patient_id)
+          failed_import = Lab::LimsFailedImport.find(failed_import_id)
+          order_dto = Lab::Lims::OrderDTO.new(lims_api.find_order(failed_import.lims_id))
+          byebug
+        end
+
+        private
+
+        def lims_api
+          @lims_api ||= Lab::Lims::Api.new
+        end
+      end
+    end
+  end
+end

data/app/services/lab/lims/migrator.rb

@@ -29,6 +29,7 @@ require 'lab/lab_test'
 require 'lab/lims_order_mapping'
 require 'lab/lims_failed_import'
 
+require_relative './config'
 require_relative './worker'
 require_relative '../orders_service'
 require_relative '../results_service'
@@ -43,53 +44,73 @@ require_relative 'utils'
 module Lab
   module Lims
     module Migrator
-      class MigratorApi < Api
-        MAX_THREADS = 6
+      MAX_THREADS = ENV.fetch('MIGRATION_WORKERS', 6).to_i
 
-        attr_reader :rejections
+      class CouchDbMigratorApi < Api::CouchDbApi
+        def initialize(*args, processes: 1, on_merge_processes: nil, **kwargs)
+          super(*args, **kwargs)
+
+          @processes = processes
+          @on_merge_processes = on_merge_processes
+        end
 
         def consume_orders(from: nil, **_kwargs)
-          limit = 50_000
+          limit = 25_000
 
-          Parallel.each(read_orders(from, limit),
-                        in_processes: MAX_THREADS,
-                        finish: order_pmap_post_processor(from)) do |row|
-            next unless row['doc']['type']&.casecmp?('Order')
+          loop do
+            on_merge_processes = ->(_item, index, _result) { @on_merge_processes&.call(from + index) }
+            processes = @processes > 1 ? @processes : 0
 
-            User.current = Utils.lab_user
-            yield OrderDTO.new(row['doc']), OpenStruct.new(last_seq: (from || 0) + limit, current_seq: from)
-          end
-        end
+            orders = read_orders(from, limit)
+            break if orders.empty?
 
-        def last_seq
-          return 0 unless File.exist?(last_seq_path)
+            Parallel.each(orders, in_processes: processes, finish: on_merge_processes) do |row|
+              next unless row['doc']['type']&.casecmp?('Order')
 
-          File.open(last_seq_path, File::RDONLY) do |file|
-            last_seq = file.read&.strip
-            return last_seq.blank? ? nil : last_seq&.to_i
+              User.current = Utils.lab_user
+              yield OrderDTO.new(row['doc']), OpenStruct.new(last_seq: (from || 0) + limit, current_seq: from)
+            end
+
+            from += orders.size
           end
         end
 
         private
 
-        def last_seq_path
-          LIMS_LOG_PATH.join('migration-last-id.dat')
+        def read_orders(from, batch_size)
+          start_key_param = from ? "&skip=#{from}" : ''
+          url = "_all_docs?include_docs=true&limit=#{batch_size}#{start_key_param}"
+
+          Rails.logger.debug("#{CouchDbMigratorApi}: Pulling orders from LIMS CouchDB: #{url}")
+          response = bum.couch_rest :get, url
+
+          response['rows']
+        end
+      end
+
+      class MigrationWorker < Worker
+        LOG_FILE_PATH = LIMS_LOG_PATH.join('migration-last-id.dat')
+
+        attr_reader :rejections
+
+        def initialize(api_class)
+          api = api_class.new(processes: MAX_THREADS, on_merge_processes: method(:save_seq))
+          super(api)
         end
 
-        def order_pmap_post_processor(last_seq)
-          lambda do |item, index, result|
-            save_last_seq(last_seq + index)
-            status, reason = result
-            next unless status == :rejected
+        def last_seq
+          return 0 unless File.exist?(LOG_FILE_PATH)
 
-            (@rejections ||= []) << OpenStruct.new(order: OrderDTO.new(item['doc']), reason: reason)
+          File.open(LOG_FILE_PATH, File::RDONLY) do |file|
+            last_seq = file.read&.strip
+            return last_seq.blank? ? nil : last_seq&.to_i
           end
         end
 
-        def save_last_seq(last_seq)
-          return unless last_seq
+        private
 
-          File.open(last_seq_path, File::WRONLY | File::CREAT, 0o644) do |file|
+        def save_seq(last_seq)
+          File.open(LOG_FILE_PATH, File::WRONLY | File::CREAT, 0o644) do |file|
             Rails.logger.debug("Process ##{Parallel.worker_number}: Saving last seq: #{last_seq}")
             file.flock(File::LOCK_EX)
             file.write(last_seq.to_s)
@@ -97,39 +118,11 @@ module Lab
           end
         end
 
-        def save_rejection(order_dto, reason); end
-
-        def read_orders(from, batch_size)
-          Enumerator.new do |enum|
-            loop do
-              start_key_param = from ? "&skip=#{from}" : ''
-              url = "_all_docs?include_docs=true&limit=#{batch_size}#{start_key_param}"
-
-              Rails.logger.debug("#{MigratorApi}: Pulling orders from LIMS CouchDB: #{url}")
-              response = bum.couch_rest :get, url
-
-              from ||= 0
-
-              break from if response['rows'].empty?
-
-              response['rows'].each do |row|
-                enum.yield(row)
-              end
-
-              from += response['rows'].size
-            end
-          end
-        end
-      end
-
-      class MigrationWorker < Worker
-        protected
+        def order_rejected(order_dto, reason)
+          @rejections ||= []
 
-        def last_seq
-          lims_api.last_seq
+          @rejections << OpenStruct.new(order: order_dto, reason: reason)
         end
-
-        def update_last_seq(_last_seq); end
       end
 
       def self.save_csv(filename, rows:, headers: nil)
@@ -177,8 +170,7 @@ module Lab
       MIGRATION_LOG_PATH = LIMS_LOG_PATH.join('migration.log')
 
       def self.start_migration
-        log_dir = Rails.root.join('log/lims')
-        Dir.mkdir(log_dir) unless File.exist?(log_dir)
+        Dir.mkdir(LIMS_LOG_PATH) unless File.exist?(LIMS_LOG_PATH)
 
         logger = LoggerMultiplexor.new(Logger.new($stdout), MIGRATION_LOG_PATH)
         logger.level = :debug
@@ -186,12 +178,17 @@ module Lab
         ActiveRecord::Base.logger = logger
         # CouchBum.logger = logger
 
-        api = MigratorApi.new
-        worker = MigrationWorker.new(api)
+        api_class = case ENV.fetch('MIGRATION_SOURCE', 'couchdb').downcase
+                    when 'couchdb' then CouchDbMigratorApi
+                    when 'mysql' then Api::MysqlApi
+                    else raise "Invalid MIGRATION_SOURCE: #{ENV['MIGRATION_SOURCE']}"
+                    end
+
+        worker = MigrationWorker.new(api_class)
 
-        worker.pull_orders
+        worker.pull_orders(batch_size: 10_000)
       ensure
-        api && export_rejections(api.rejections)
+        worker && export_rejections(worker.rejections)
         export_failures
       end
     end

data/app/services/lab/lims/order_dto.rb

@@ -69,9 +69,7 @@ module Lab
       end
 
       def start_date
-        if self['date_created'].blank?
-          raise LimsException, 'Order missing created date'
-        end
+        raise LimsException, 'Order missing created date' if self['date_created'].blank?
 
         Utils.parse_date(self['date_created'])
       end

data/app/services/lab/lims/order_serializer.rb

@@ -24,6 +24,8 @@ module Lab
             sample_type: format_sample_type(serialized_order.specimen.name),
             sample_status: format_sample_status(serialized_order.specimen.name),
             sample_statuses: format_sample_status_trail(order),
+            test_statuses: format_test_status_trail(order),
+            who_order_test: format_orderer(order),
             districy: current_district, # yes districy [sic]...
             priority: serialized_order.reason_for_test.name,
             date_created: serialized_order.order_date,
@@ -75,9 +77,7 @@ module Lab
         end
 
         def format_sample_status_trail(order)
-          if order.concept_id == ConceptName.find_by_name!('Unknown').concept_id
-            return []
-          end
+          return [] if order.concept_id == ConceptName.find_by_name!('Unknown').concept_id
 
           user = User.find(order.discontinued_by || order.creator)
           drawn_by = PersonName.find_by_person_id(user.user_id)
@@ -96,6 +96,31 @@ module Lab
           ]
         end
 
+        def format_test_status_trail(order)
+          order.tests.each_with_object({}) do |test, trail|
+            test_name = ConceptName.find_by_concept_id!(test.value_coded).name
+            test_name = 'Viral load' if test_name.casecmp?('HIV Viral Load')
+
+            current_test_trail = trail[test_name] = {}
+
+            current_test_trail[test.obs_datetime.strftime('%Y%m%d%H%M%S')] = {
+              status: 'Drawn',
+              updated_by: find_user(test.creator)
+            }
+
+            next unless test.result
+
+            current_test_trail[test.obs_datetime.strftime('%Y%m%d%H%M%S')] = {
+              status: 'Verified',
+              updated_by: find_user(test.result.creator)
+            }
+          end
+        end
+
+        def format_orderer(order)
+          find_user(order.creator)
+        end
+
         def format_test_results(order)
           order.tests&.each_with_object({}) do |test, results|
             next unless test.result
@@ -134,6 +159,20 @@ module Lab
         def current_facility_name
           current_health_center.name
         end
+
+        def find_user(user_id)
+          user = User.find(user_id)
+          person_name = PersonName.find_by(person_id: user.person_id)
+          phone_number = PersonAttribute.find_by(type: PersonAttributeType.where(name: 'Cell phone number'),
+                                                 person_id: user.person_id)
+
+          {
+            first_name: person_name&.given_name,
+            last_name: person_name&.family_name,
+            phone_number: phone_number&.value,
+            id: user.username
+          }
+        end
       end
     end
   end

data/app/services/lab/lims/utils.rb

@@ -52,21 +52,22 @@ module Lab
       end
 
       def self.parse_date(str_date, fallback_date = nil)
-        if str_date.blank? && fallback_date.blank?
-          raise "Can't parse blank date"
-        end
+        str_date = str_date&.to_s
+
+        raise "Can't parse blank date" if str_date.blank? && fallback_date.blank?
 
         return parse_date(fallback_date) if str_date.blank?
 
         str_date = str_date.gsub(/^00/, '20').gsub(/^180/, '20')
 
-        if str_date.match?(/\d{4}-\d{2}-\d{2}/)
+        case str_date
+        when /\d{4}-\d{2}-\d{2}/
           str_date
-        elsif str_date.match?(/\d{2}-\d{2}-\d{2}/)
+        when /\d{2}-\d{2}-\d{2}/
           Date.strptime(str_date, '%d-%m-%Y').strftime('%Y-%m-%d')
-        elsif str_date.match?(/(\d{4}\d{2}\d{2})\d+/)
+        when /(\d{4}\d{2}\d{2})\d+/
           Date.strptime(str_date, '%Y%m%d').strftime('%Y-%m-%d')
-        elsif str_date.match?(%r{\d{2}/\d{2}/\d{4}})
+        when %r{\d{2}/\d{2}/\d{4}}
           str_date.to_date.to_s
         else
           Rails.logger.warn("Invalid date: #{str_date}")

data/app/services/lab/lims/worker.rb

@@ -2,7 +2,7 @@
 
 require 'cgi/util'
 
-require_relative './api'
+require_relative './api/couchdb_api'
 require_relative './exceptions'
 require_relative './order_serializer'
 require_relative './utils'
@@ -26,11 +26,11 @@ module Lab
           User.current = Utils.lab_user
 
           fout.write("Worker ##{Process.pid} started at #{Time.now}")
-          worker = new(Api.new)
+          worker = new(Api::CouchDbApi.new)
           worker.pull_orders
           # TODO: Verify that names being pushed to LIMS are of the correct format (ie matching
           # LIMS naming conventions). Enable pushing when that is done
-          # worker.push_orders
+          worker.push_orders
         end
       end
 
@@ -81,28 +81,34 @@ module Lab
 
       ##
       # Pulls orders from the LIMS queue and writes them to the local database
-      def pull_orders
+      def pull_orders(batch_size: 10_000)
         logger.info("Retrieving LIMS orders starting from #{last_seq}")
 
-        lims_api.consume_orders(from: last_seq, limit: 100) do |order_dto, context|
+        lims_api.consume_orders(from: last_seq, limit: batch_size) do |order_dto, context|
           logger.debug("Retrieved order ##{order_dto[:tracking_number]}: #{order_dto}")
 
           patient = find_patient_by_nhid(order_dto[:patient][:id])
           unless patient
             logger.debug("Discarding order: Non local patient ##{order_dto[:patient][:id]} on order ##{order_dto[:tracking_number]}")
-            next [:rejected, "Patient NPID, '#{order_dto[:patient][:id]}', didn't match any local NPIDs"]
+            order_rejected(order_dto, "Patient NPID, '#{order_dto[:patient][:id]}', didn't match any local NPIDs")
+            next
+          end
+
+          if order_dto[:tests].empty?
+            logger.debug("Discarding order: Missing tests on order ##{order_dto[:tracking_number]}")
+            order_rejected(order_dto, 'Order is missing tests')
+            next
           end
 
           diff = match_patient_demographics(patient, order_dto['patient'])
           if diff.empty?
             save_order(patient, order_dto)
+            order_saved(order_dto)
           else
             save_failed_import(order_dto, 'Demographics not matching', diff)
           end
 
           update_last_seq(context.current_seq)
-
-          [:accepted, "Patient NPID, '#{order_dto[:patient][:id]}', matched"]
         rescue DuplicateNHID
           logger.warn("Failed to import order due to duplicate patient NHID: #{order_dto[:patient][:id]}")
           save_failed_import(order_dto, "Duplicate local patient NHID: #{order_dto[:patient][:id]}")
@@ -134,23 +140,33 @@ module Lab
         end
       end
 
+      def order_saved(order_dto); end
+
+      def order_rejected(order_dto, message); end
+
       private
 
       def find_patient_by_nhid(nhid)
         national_id_type = PatientIdentifierType.where(name: ['National id', 'Old Identification Number'])
         identifiers = PatientIdentifier.where(type: national_id_type, identifier: nhid)
+                                       .joins('INNER JOIN person ON person.person_id = patient_identifier.patient_id AND person.voided  = 0')
         if identifiers.count.zero?
-          identifiers = PatientIdentifier.unscoped.where(voided: 1, type: national_id_type, identifier: nhid)
+          identifiers = PatientIdentifier.unscoped
+                                         .where(voided: 1, type: national_id_type, identifier: nhid)
+                                         .joins('INNER JOIN person ON person.person_id = patient_identifier.patient_id AND person.voided  = 0')
         end
+
+        # Joining to person above to ensure that the person is not voided,
+        # it was noted at one site that there were some people that were voided
+        # upon merging but the patient and patient_identifier was not voided
+
         return nil if identifiers.count.zero?
 
         patients = Patient.where(patient_id: identifiers.select(:patient_id))
                           .distinct(:patient_id)
                           .all
 
-        if patients.size > 1
-          raise DuplicateNHID, "Duplicate National Health ID: #{nhid}"
-        end
+        raise DuplicateNHID, "Duplicate National Health ID: #{nhid}" if patients.size > 1
 
         patients.first
       end
@@ -201,10 +217,11 @@ module Lab
             mapping.update(pulled_at: Time.now)
           else
             order = create_order(patient, order_dto)
-            LimsOrderMapping.create!(lims_id: order_dto[:_id],
-                                     order_id: order['id'],
-                                     pulled_at: Time.now,
-                                     revision: order_dto['_rev'])
+            mapping = LimsOrderMapping.create(lims_id: order_dto[:_id],
+                                              order_id: order['id'],
+                                              pulled_at: Time.now,
+                                              revision: order_dto['_rev'])
+            byebug unless mapping.errors.empty?
           end
 
           order
@@ -214,9 +231,7 @@ module Lab
       def create_order(patient, order_dto)
         logger.debug("Creating order ##{order_dto['_id']}")
         order = OrdersService.order_test(order_dto.to_order_service_params(patient_id: patient.patient_id))
-        unless order_dto['test_results'].empty?
-          update_results(order, order_dto['test_results'])
-        end
+        update_results(order, order_dto['test_results']) unless order_dto['test_results'].empty?
 
         order
       end
@@ -225,9 +240,7 @@ module Lab
         logger.debug("Updating order ##{order_dto['_id']}")
         order = OrdersService.update_order(order_id, order_dto.to_order_service_params(patient_id: patient.patient_id)
                                                               .merge(force_update: true))
-        unless order_dto['test_results'].empty?
-          update_results(order, order_dto['test_results'])
-        end
+        update_results(order, order_dto['test_results']) unless order_dto['test_results'].empty?
 
         order
       end
@@ -242,6 +255,8 @@ module Lab
             next
           end
 
+          next unless test_results['results']
+
           measures = test_results['results'].map do |indicator, value|
             measure = find_measure(order, indicator, value)
             next nil unless measure
@@ -250,7 +265,6 @@ module Lab
           end
 
           measures = measures.compact
-
           next if measures.empty?
 
           creator = format_result_entered_by(test_results['result_entered_by'])

data/app/services/lab/results_service.rb

@@ -87,9 +87,7 @@ module Lab
       end
 
       def validate_measure_params(params)
-        if params[:value].blank?
-          raise InvalidParameterError, 'measures.value is required'
-        end
+        raise InvalidParameterError, 'measures.value is required' if params[:value].blank?
 
         if params[:indicator]&.[](:concept_id).blank?
           raise InvalidParameterError, 'measures.indicator.concept_id is required'

data/lib/lab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lab
-  VERSION = '0.0.11'
+  VERSION = '1.0.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: his_emr_api_lab
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 1.0.0
 platform: ruby
 authors:
 - Elizabeth Glaser Pediatric Foundation Malawi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-29 00:00:00.000000000 Z
+date: 2021-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: couchrest
@@ -249,9 +249,11 @@ files:
 - app/services/lab/accession_number_service.rb
 - app/services/lab/concepts_service.rb
 - app/services/lab/labelling_service/order_label.rb
-- app/services/lab/lims/api.rb
+- app/services/lab/lims/api/couchdb_api.rb
+- app/services/lab/lims/api/mysql_api.rb
 - app/services/lab/lims/config.rb
 - app/services/lab/lims/exceptions.rb
+- app/services/lab/lims/failed_imports.rb
 - app/services/lab/lims/migrator.rb
 - app/services/lab/lims/order_dto.rb
 - app/services/lab/lims/order_serializer.rb

data/app/services/lab/lims/api.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-require 'couch_bum/couch_bum'
-
-require_relative './config'
-
-module Lab
-  module Lims
-    ##
-    # Talk to LIMS like a boss
-    class Api
-      attr_reader :bum
-
-      def initialize(config: nil)
-        config ||= Config.couchdb
-
-        @bum = CouchBum.new(protocol: config['protocol'],
-                            host: config['host'],
-                            port: config['port'],
-                            database: "#{config['prefix']}_order_#{config['suffix']}",
-                            username: config['username'],
-                            password: config['password'])
-      end
-
-      ##
-      # Consume orders from the LIMS queue.
-      #
-      # Retrieves orders from the LIMS queue and passes each order to
-      # given block until the queue is empty or connection is terminated
-      # by calling method +choke+.
-      def consume_orders(from: 0, limit: 30)
-        bum.binge_changes(since: from, limit: limit, include_docs: true) do |change|
-          next unless change['doc']['type']&.casecmp?('Order')
-
-          yield OrderDTO.new(change['doc']), self
-        end
-      end
-
-      def create_order(order)
-        bum.couch_rest :post, '/', order
-      end
-
-      def update_order(id, order)
-        bum.couch_rest :put, "/#{id}", order
-      end
-    end
-  end
-end