his_emr_api_lab 2.1.9.pre.beta → 2.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6c20f1ba07edd60ba5795eaaef1ca3f34395060b1f9ed0160a5513777d657d0
-  data.tar.gz: d16b695b3592233474fa9bbc2f90c85db7d04161a02f8107c56f92fd4e82a9fb
+  metadata.gz: 5430975d278d3f06b169b906ca5c471f0ad632dc158540f358545f4da2aa05d1
+  data.tar.gz: e75574b9ac3023e974d70ef021ae158ded0614a8d379eec120124f9e45891479
 SHA512:
-  metadata.gz: 035cf4578ee6f60ba3406a66f88ea33cc54ea7d85ab38524b7e6ae54c9b4b80e03fe2938fc34d047e72df9423b71b8eba1f40004a8fd688555b56c3aebb81605
-  data.tar.gz: feda70dff8797923f46078a42429b274d7562eb5105a5d43c1d746f744436c6c516847aa2ef68d7b235f1d1a216981e10d4c707bc6f4fb45372d6b867c412a2a
+  metadata.gz: b3f869dd69e746a19f35be7741f717c1b657ae6d8df6a55cf72a427f70fbdee62c76d61af605ac81f248a376428db8fab67bd6ef9fa623ceb2ca0bec45678cd6
+  data.tar.gz: a7c3ef0003f803d83f47337f3d4ba4f9388a4b40962f6b95b36d14c904ab361280971e6689f8fcbb3cd48f41ba4863722d422c6d884e44e346f0e48626cacedc

data/README.md

@@ -22,25 +22,25 @@ For details on how to perform these operations please see the
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'lab', git: 'https://github.com/EGPAFMalawiHIS/HIS-EMR-API-Lab', branch: 'development'
+gem 'his_emr_api_lab', git: 'https://github.com/EGPAFMalawiHIS/HIS-EMR-API-Lab', branch: 'development'
 ```
 
 And then execute:
 
 ```bash
-$ bundle install lab
+$ bundle install 
 ```
 
 Or install it yourself as:
 
 ```bash
-$ gem install lab
+$ gem install his_emr_api_lab
 ```
 
 Finally run:
 
 ```bash
-$ bundle exec rails lab:install
+$ bundle exec rails his_emr_api_lab:install
 ```
 
 ## Configuration
@@ -66,6 +66,19 @@ but too much a departure from it is frowned upon. For example, you will be forgi
 for writing a method with 15 to 20 lines if you clearly justify why you couldn't
 break that method into multiple smaller methods.
 
+## Publishing
+
+To publish a new version of the gem, first update the version number in
+`lib/lab/version.rb` and then run the following command:
+
+```bash
+$ gem build his_emr_api_lab.gemspec
+$ gem push his_emr_api_lab-<version>.gem  
+```
+Make sure to replace `<version>` with the version number you set in `lib/lab/version.rb`.
+
+NB: You need to have an account on [rubygems.org](https://rubygems.org/) and permission to publish gems.
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/app/controllers/lab/orders_controller.rb

@@ -2,6 +2,7 @@
 
 module Lab
   class OrdersController < ApplicationController
+    skip_before_action :authenticate, only: %i[order_status order_result summary]
     before_action :authenticate_request, only: %i[order_status order_result summary]
 
     def create
@@ -52,7 +53,8 @@ module Lab
     end
 
     def order_status
-      order_params = params.permit(:tracking_number, :status, :status_time, :comments)
+      order_params = params.permit(:tracking_number, :status, :status_time, :comments, :status_id,
+                                    updated_by: [:first_name, :last_name, :id, :phone_number])
       OrdersService.update_order_status(order_params)
       render json: { message: "Status for order #{order_params['tracking_number']} successfully updated" }, status: :ok
     end
@@ -76,8 +78,26 @@ module Lab
     private
 
     def authenticate_request
-      decoded_user = authorize_request
-      user(decoded_user)
+      header = request.headers['Authorization']
+      content = header.split(' ')
+      auth_scheme = content.first
+      unless header
+        errors = ['Authorization token required']
+        render json: { errors: errors }, status: :unauthorized
+        return false
+      end
+      unless auth_scheme == 'Bearer'
+        errors = ['Authorization token bearer scheme required']
+        render json: { errors: errors }, status: :unauthorized
+        return false
+      end
+      process_token(content.last)
+    end
+
+    def process_token(token)
+      browser = Browser.new(request.user_agent)
+      decoded = Lab::JsonWebTokenService.decode(token, request.remote_ip + browser.name + browser.version)
+      user(decoded)
     end
 
     def user(decoded)

data/app/models/lab/lab_order.rb

@@ -8,6 +8,11 @@ module Lab
 
         -> { where(concept:) }
       end
+
+      # Cache the concept ID to avoid lookups in association scopes
+      def order_status_concept_id
+        @order_status_concept_id ||= ConceptName.find_by(name: 'Lab Order Status')&.concept_id
+      end
     end
 
     has_many :tests,
@@ -44,12 +49,20 @@ module Lab
             class_name: '::Lab::LimsOrderMapping',
             foreign_key: :order_id
 
+    # Status trails are stored as observations with concept 'Lab Order Status'
+    has_many :status_trail_observations,
+             lambda {
+               unscoped.where(voided: 0, concept_id: Lab::LabOrder.order_status_concept_id).order(obs_datetime: :asc)
+             },
+             class_name: 'Observation',
+             foreign_key: :order_id
+
     default_scope do
       joins(:order_type)
         .merge(OrderType.where(name: [
-          Lab::Metadata::ORDER_TYPE_NAME, 
-          Lab::Metadata::HTS_ORDER_TYPE_NAME
-        ]))
+                                 Lab::Metadata::ORDER_TYPE_NAME,
+                                 Lab::Metadata::HTS_ORDER_TYPE_NAME
+                               ]))
         .where.not(concept_id: ConceptName.where(name: 'Tests ordered').select(:concept_id))
     end
 
@@ -57,11 +70,13 @@ module Lab
     scope :not_drawn, -> { where(concept_id: ConceptName.where(name: 'Unknown').select(:concept_id)) }
 
     def self.prefetch_relationships
-      includes(:reason_for_test,
-               :requesting_clinician,
-               :target_lab,
-               :comment_to_fulfiller,
-               tests: [:result])
+      # NOTE: status_trail_observations and test results are not preloaded due to
+      # Rails limitations with eager loading unscoped associations. They load on-demand instead.
+      preload(:reason_for_test,
+              :requesting_clinician,
+              :target_lab,
+              :comment_to_fulfiller,
+              :tests)
     end
   end
 end

data/app/models/lab/lab_result.rb

@@ -3,9 +3,9 @@
 module Lab
   class LabResult < Observation
     def children
-      Observation.where(obs_group_id: obs_id, voided: 0)
+      Observation.unscoped.where(obs_group_id: obs_id, voided: 0)
     end
-    
+
     alias measures children
 
     default_scope do

data/app/models/lab/lab_test.rb

@@ -6,14 +6,33 @@ module Lab
       where(concept: ConceptName.where(name: Lab::Metadata::TEST_TYPE_CONCEPT_NAME))
     end
 
+    # Cache the concept IDs as class methods to avoid lookups in association scopes
+    def self.test_status_concept_id
+      @test_status_concept_id ||= ConceptName.find_by(name: 'Lab Test Status')&.concept_id
+    end
+
+    def self.test_result_concept_id
+      @test_result_concept_id ||= ConceptName.find_by(name: Lab::Metadata::TEST_RESULT_CONCEPT_NAME)&.concept_id
+    end
+
     has_one :result,
-            -> { where(concept: ConceptName.where(name: Lab::Metadata::TEST_RESULT_CONCEPT_NAME)) },
+            -> { unscoped.where(voided: 0, concept_id: Lab::LabTest.test_result_concept_id) },
             class_name: 'Lab::LabResult',
             foreign_key: :obs_group_id
 
+    # Status trails are stored as observations with concept 'Lab Test Status'
+    # They are linked via obs_group_id (this test obs is the parent)
+    has_many :status_trail_observations,
+             lambda {
+               unscoped.where(voided: 0, concept_id: Lab::LabTest.test_status_concept_id).order(obs_datetime: :asc)
+             },
+             class_name: 'Observation',
+             foreign_key: :obs_group_id,
+             primary_key: :obs_id
+
     def void(reason)
       result&.void(reason)
-      super(reason)
+      super
     end
   end
 end

data/app/serializers/lab/lab_order_serializer.rb

@@ -9,9 +9,8 @@ module Lab
       reason_for_test ||= order.reason_for_test
       target_lab = target_lab&.value_text || order.target_lab&.value_text || Location.current_health_center&.name
 
-      # Use unscoped to find encounter across all locations
-      encounter = Encounter.unscoped.find_by_encounter_id(order.encounter_id)
-      program = Program.find_by_program_id(encounter.program_id)
+      encounter = Encounter.find_by_encounter_id(order.encounter_id)
+      program = Program.find_by_program_id(encounter&.program_id)
 
       ActiveSupport::HashWithIndifferentAccess.new(
         {
@@ -20,8 +19,8 @@ module Lab
           order_id: order.order_id, # Deprecated: Link to :id
           encounter_id: order.encounter_id,
           order_date: order.start_date,
-          location_id: encounter.location_id,
-          program_id: encounter.program_id,
+          location_id: encounter&.location_id,
+          program_id: encounter&.program_id,
           program_name: program&.name,
           patient_id: order.patient_id,
           accession_number: order.accession_number,
@@ -37,8 +36,10 @@ module Lab
             name: concept_name(reason_for_test&.value_coded)
           },
           delivery_mode: order&.lims_acknowledgement_status&.acknowledgement_type,
+          order_status: latest_order_status(order),
+          order_status_trail: serialize_order_status_trail(order),
           tests: tests.map do |test|
-            result_obs = test.children.first
+            result_obs = test.result
 
             {
               id: test.obs_id,
@@ -46,7 +47,9 @@ module Lab
               uuid: test.uuid,
               name: concept_name(test.value_coded),
               test_method: test_method(order, test.value_coded),
-              result: result_obs && ResultSerializer.serialize(result_obs)
+              result: result_obs && ResultSerializer.serialize(result_obs),
+              test_status: latest_test_status(test),
+              test_status_trail: serialize_test_status_trail(test)
             }
           end
         }
@@ -75,5 +78,72 @@ module Lab
                            .select(:concept_id)
       LabTest.unscoped.where(concept:, order:, voided: true)
     end
+
+    def self.latest_order_status(order)
+      # Query obs table for latest order status
+      latest_obs = order.status_trail_observations.last
+      return nil unless latest_obs
+
+      updated_by = parse_comments_json(latest_obs.comments)
+
+      {
+        status_id: 0, # status_id not used with text values
+        status: latest_obs.value_text,
+        timestamp: latest_obs.obs_datetime,
+        updated_by: updated_by
+      }
+    end
+
+    def self.serialize_order_status_trail(order)
+      # Query obs table for order status trail
+      order.status_trail_observations.map do |obs|
+        updated_by = parse_comments_json(obs.comments)
+
+        {
+          status_id: 0, # status_id not used with text values
+          status: obs.value_text,
+          timestamp: obs.obs_datetime,
+          updated_by: updated_by
+        }
+      end
+    end
+
+    def self.latest_test_status(test)
+      # Query obs table for latest test status
+      latest_obs = test.status_trail_observations.last
+      return nil unless latest_obs
+
+      updated_by = parse_comments_json(latest_obs.comments)
+
+      {
+        status_id: 0, # status_id not used with text values
+        status: latest_obs.value_text,
+        timestamp: latest_obs.obs_datetime,
+        updated_by: updated_by
+      }
+    end
+
+    def self.serialize_test_status_trail(test)
+      # Query obs table for test status trail
+      test.status_trail_observations.map do |obs|
+        updated_by = parse_comments_json(obs.comments)
+
+        {
+          status_id: 0, # status_id not used with text values
+          status: obs.value_text,
+          timestamp: obs.obs_datetime,
+          updated_by: updated_by
+        }
+      end
+    end
+
+    # Helper to parse updated_by from obs comments field
+    def self.parse_comments_json(comments)
+      return {} if comments.blank?
+
+      JSON.parse(comments)
+    rescue JSON::ParserError
+      {}
+    end
   end
 end

data/app/serializers/lab/result_serializer.rb

@@ -11,9 +11,9 @@ module Lab
         concept_name = get_test_catalog_concept_name(measure.concept_id)
         program_id = ''
         if measure.obs_id.present?
-          obs = Observation.unscope(where: :obs_group_id).find(measure.obs_id)
-          encounter = Encounter.find(obs.encounter_id)
-          program_id = encounter.program_id
+          obs = Observation.unscope(where: :obs_group_id).find_by(obs_id: measure.obs_id)
+          encounter = Encounter.find_by(encounter_id: obs&.encounter_id)
+          program_id = encounter&.program_id
         end
 
         {

data/app/services/lab/acknowledgement_service.rb

@@ -14,9 +14,12 @@ module Lab
                                         date_received: params[:date_received])
       end
 
-      def acknowledgements_pending_sync(batch_size)
-        Lab::LabAcknowledgement.where(pushed: false)
-                               .limit(batch_size)
+      def acknowledgements_pending_sync(batch_size, start_date: nil)
+        query = Lab::LabAcknowledgement.joins(:order).where(pushed: false)
+
+        query = query.where('orders.date_created >= ?', start_date) if start_date
+
+        query.limit(batch_size)
       end
 
       def push_acknowledgement(acknowledgement, lims_api)
@@ -30,7 +33,8 @@ module Lab
             Rails.logger.info("Updating acknowledgement ##{acknowledgement_dto[:tracking_number]} in LIMS")
             response = lims_api.acknowledge(acknowledgement_dto)
             Rails.logger.info("Info #{response}")
-            if ['results already delivered for test name given', 'test result acknowledged successfully', 'test result already acknowledged electronically at facility'].include?(response['message'])
+            if ['results already delivered for test name given', 'test result acknowledged successfully',
+                'test result already acknowledged electronically at facility'].include?(response['message'])
               acknowledgement.pushed = true
               acknowledgement.date_pushed = Time.now
               acknowledgement.save!

data/app/services/lab/lims/acknowledgement_worker.rb

@@ -4,20 +4,22 @@ module Lab
   module Lims
     # This class is responsible for handling the acknowledgement of lab orders
     class AcknowledgementWorker
-      attr_reader :lims_api
+      attr_reader :lims_api, :start_date
 
       include Utils # for logger
 
       SECONDS_TO_WAIT_FOR_ORDERS = 30
 
-      def initialize(lims_api)
+      def initialize(lims_api, start_date: nil)
         @lims_api = lims_api
+        @start_date = start_date
       end
 
       def push_acknowledgement(batch_size: 1000, wait: false)
         loop do
           logger.info('Looking for new acknowledgements to push to LIMS...')
-          acknowledgements = Lab::AcknowledgementService.acknowledgements_pending_sync(batch_size).all
+          acknowledgements = Lab::AcknowledgementService.acknowledgements_pending_sync(batch_size,
+                                                                                       start_date: start_date).all
 
           logger.debug("Found #{acknowledgements.size} acknowledgements...")
           acknowledgements.each do |acknowledgement|

data/app/services/lab/lims/api/rest_api.rb

@@ -74,18 +74,85 @@ module Lab
           { tracking_number: order_dto[:tracking_number] }
         end
 
-        def consume_orders(*_args, patient_id: nil, **_kwargs)
-          orders_pending_updates(patient_id).each do |order|
+        def consume_orders(*_args, patient_id: nil, start_date: nil, **_kwargs)
+          orders_pending_updates(patient_id, start_date: start_date).each do |order|
             order_dto = Lab::Lims::OrderSerializer.serialize_order(order)
-            if order_dto['priority'].nil? || order_dto['sample_type'].casecmp?('not_specified')
-              patch_order_dto_with_lims_order!(order_dto, find_lims_order(order.accession_number))
+
+            # Always fetch the full order from NLIMS to get status trails
+            begin
+              lims_order = find_lims_order(order.accession_number)
+              patch_order_dto_with_lims_order!(order_dto, lims_order)
+
+              Rails.logger.debug("NLIMS order structure for #{order.accession_number}:")
+              Rails.logger.debug("  Has 'order' key: #{lims_order.key?('order')}")
+              Rails.logger.debug("  Has 'data' key: #{lims_order.key?('data')}")
+              Rails.logger.debug("  Top level keys: #{lims_order.keys.inspect}")
+
+              # Also extract status trails from the NLIMS order
+              # Note: NLIMS might return order data under 'order' or 'data.order'
+              order_data = lims_order['order'] || lims_order.dig('data', 'order') || lims_order
+
+              if order_data && order_data['status_trail']
+                Rails.logger.info("Found #{order_data['status_trail'].size} order status trail entries from NLIMS")
+                order_dto[:sample_statuses] ||= []
+                # Convert NLIMS status trail to the format expected by PullWorker
+                # Note: sample_statuses must be an array of single-key hashes
+                order_data['status_trail'].each do |trail|
+                  # Convert ISO 8601 timestamp to YYYYMMDDHHmmss format
+                  timestamp_key = convert_timestamp_to_key(trail['timestamp'])
+                  order_dto[:sample_statuses] << {
+                    timestamp_key => {
+                      'status_id' => trail['status_id'],
+                      'status' => trail['status'],
+                      'updated_by' => trail['updated_by']
+                    }
+                  }
+                  Rails.logger.debug("  Added order status: #{trail['status']} at #{timestamp_key}")
+                end
+                Rails.logger.debug("Final sample_statuses: #{order_dto[:sample_statuses].inspect}")
+              else
+                Rails.logger.warn("No order status_trail found in NLIMS response for #{order.accession_number}")
+                Rails.logger.debug("Order data keys: #{order_data&.keys&.inspect}")
+              end
+
+              # Extract test status trails from NLIMS tests
+              tests_data = lims_order['tests'] || lims_order.dig('data', 'tests') || []
+              if tests_data.is_a?(Array)
+                Rails.logger.debug("Processing #{tests_data.size} tests from NLIMS")
+                order_dto['test_statuses'] ||= {}
+                tests_data.each do |test|
+                  next unless test['status_trail'].is_a?(Array)
+
+                  test_name = test.dig('test_type', 'name')
+                  next unless test_name
+
+                  Rails.logger.debug("  Found #{test['status_trail'].size} status trail entries for test #{test_name}")
+                  order_dto['test_statuses'][test_name] ||= {}
+                  test['status_trail'].each do |trail|
+                    # Convert ISO 8601 timestamp to YYYYMMDDHHmmss format
+                    timestamp_key = convert_timestamp_to_key(trail['timestamp'])
+                    order_dto['test_statuses'][test_name][timestamp_key] = {
+                      'status_id' => trail['status_id'],
+                      'status' => trail['status'],
+                      'updated_by' => trail['updated_by']
+                    }
+                  end
+                end
+              end
+            rescue RestClient::NotFound
+              Rails.logger.warn("Order ##{order.accession_number} not found in NLIMS, using local data only")
             end
+
+            # Try to fetch results if available
             if order_dto['test_results'].empty?
               begin
                 patch_order_dto_with_lims_results!(order_dto, find_lims_results(order.accession_number))
-              rescue InvalidParameters => e # LIMS responds with a 401 when a result is not found :(
-                Rails.logger.error("Failed to fetch results for ##{order.accession_number}: #{e.message}")
-                next
+              rescue InvalidParameters => e
+                Rails.logger.info("No results available for ##{order.accession_number}: #{e.message}")
+                # Don't skip - continue processing to save status trails
+              rescue RestClient::NotFound
+                Rails.logger.info("No results found for ##{order.accession_number}")
+                # Don't skip - continue processing to save status trails
               end
             end
 
@@ -517,46 +584,64 @@ module Lab
           }
         end
 
-        def orders_pending_updates(patient_id = nil)
+        def orders_pending_updates(patient_id = nil, start_date: nil)
           Rails.logger.info('Looking for orders that need to be updated...')
           orders = {}
 
-          orders_without_specimen(patient_id).each { |order| orders[order.order_id] = order }
-          orders_without_results(patient_id).each { |order| orders[order.order_id] = order }
-          orders_without_reason(patient_id).each { |order| orders[order.order_id] = order }
+          orders_without_specimen(patient_id, start_date: start_date).each { |order| orders[order.order_id] = order }
+          orders_without_results(patient_id, start_date: start_date).each { |order| orders[order.order_id] = order }
+          orders_without_reason(patient_id, start_date: start_date).each { |order| orders[order.order_id] = order }
 
           orders.values
         end
 
-        def orders_without_specimen(patient_id = nil)
+        def orders_without_specimen(patient_id = nil, start_date: nil)
           Rails.logger.debug('Looking for orders without a specimen')
           unknown_specimen = ConceptName.where(name: Lab::Metadata::UNKNOWN_SPECIMEN)
                                         .select(:concept_id)
           orders = Lab::LabOrder.where(concept_id: unknown_specimen)
                                 .where.not(accession_number: Lab::LimsOrderMapping.select(:lims_id))
           orders = orders.where(patient_id:) if patient_id
+          orders = orders.where('orders.date_created >= ?', start_date) if start_date
 
           orders
         end
 
-        def orders_without_results(patient_id = nil)
+        def orders_without_results(patient_id = nil, start_date: nil)
           Rails.logger.debug('Looking for orders without a result')
           # Lab::OrdersSearchService.find_orders_without_results(patient_id: patient_id)
           #                         .where.not(accession_number: Lab::LimsOrderMapping.select(:lims_id).where("pulled_at IS NULL"))
-          Lab::OrdersSearchService.find_orders_without_results(patient_id:)
-                                  .where(order_id: Lab::LimsOrderMapping.select(:order_id))
+          orders = Lab::OrdersSearchService.find_orders_without_results(patient_id:)
+                                           .where(order_id: Lab::LimsOrderMapping.select(:order_id))
+          orders = orders.where('orders.date_created >= ?', start_date) if start_date
+          orders
         end
 
-        def orders_without_reason(patient_id = nil)
+        def orders_without_reason(patient_id = nil, start_date: nil)
           Rails.logger.debug('Looking for orders without a reason for test')
           orders = Lab::LabOrder.joins(:reason_for_test)
                                 .merge(Observation.where(value_coded: nil, value_text: nil))
                                 .limit(1000)
                                 .where.not(accession_number: Lab::LimsOrderMapping.select(:lims_id))
           orders = orders.where(patient_id:) if patient_id
+          orders = orders.where('orders.date_created >= ?', start_date) if start_date
 
           orders
         end
+
+        # Converts ISO 8601 timestamp to YYYYMMDDHHmmss format
+        def convert_timestamp_to_key(timestamp)
+          return timestamp if timestamp.nil? || timestamp.empty?
+
+          begin
+            # Parse ISO 8601 timestamp and format as YYYYMMDDHHmmss
+            Time.parse(timestamp).strftime('%Y%m%d%H%M%S')
+          rescue StandardError => e
+            Rails.logger.warn("Failed to parse timestamp '#{timestamp}': #{e.message}")
+            # Fallback: remove all non-digits
+            timestamp.to_s.gsub(/\D/, '')
+          end
+        end
       end
     end
   end