his_emr_api_lab 2.1.8 → 2.1.9.pre.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44e72036503453b83d8f3cfcdff281e1085e461c476a8f793e24a3d72f12f223
-  data.tar.gz: 0100d95046a1a81db0fc8b7eace3e9fa02dc2f9be6f396f073954bbfa5578993
+  metadata.gz: cfef9418fc9d94d233f24d04883623d67e0c59c2aacea66bc15d3cca61192f78
+  data.tar.gz: 2d6324bf11f30c87398819042e3a0835275596961bfd838d088a63ae7a4f93c2
 SHA512:
-  metadata.gz: 25cb61dabafa209680d649747e35a06e19ed3685c7730bc0c539a35357e9e9529794ed180714c626125ec6ab8104b107e90e6ad0ab0cbebfcaee2ae728ea7815
-  data.tar.gz: 0da64fc5aa9647ffcc3252ae597b0492bacb3b77ec97064c4433f72bcd122431e04114323f7f13dc602faa0ccdc7c2f836b098a1090e9a56fa8cae774503101f
+  metadata.gz: c7eaa989ae6027aaba3bcd29e90136d1f93eda66854f7ee5bf84453420e0c66a40a928e460e00ee48bdee3fac8e2de05b7c030a7c71c8d4a6bf9457704dc7c6f
+  data.tar.gz: 16048c52a25d2876d79d6b3526b13625c67de47c44717de48910f905a75bb02dbd1cd48d7ce22e8b58947e61a1cb603340ea3bbdb1cce7536bb1774f41a72d65

data/app/controllers/lab/orders_controller.rb

@@ -52,7 +52,8 @@ module Lab
     end
 
     def order_status
-      order_params = params.permit(:tracking_number, :status, :status_time, :comments)
+      order_params = params.permit(:tracking_number, :status, :status_time, :comments, :status_id,
+                                    updated_by: [:first_name, :last_name, :id, :phone_number])
       OrdersService.update_order_status(order_params)
       render json: { message: "Status for order #{order_params['tracking_number']} successfully updated" }, status: :ok
     end

data/app/jobs/lab/process_lab_result_job.rb

@@ -7,7 +7,7 @@ module Lab
     queue_as :default
     def perform(results_obs_id, serializer, result_enter_by)
       Rails.logger.info("Lab::ProcessLabResultJob: Processing result completion for #{serializer}")
-      results_obs = Lab::LabResult.find(results_obs_id)
+      results_obs = Lab::LabResult.unscoped.find(results_obs_id)
       Lab::ResultsService.process_result_completion(results_obs, serializer, result_enter_by)
     end
   end

data/app/models/lab/lab_order.rb

@@ -8,6 +8,11 @@ module Lab
 
         -> { where(concept:) }
       end
+
+      # Cache the concept ID to avoid lookups in association scopes
+      def order_status_concept_id
+        @order_status_concept_id ||= ConceptName.find_by(name: 'Lab Order Status')&.concept_id
+      end
     end
 
     has_many :tests,
@@ -44,12 +49,20 @@ module Lab
             class_name: '::Lab::LimsOrderMapping',
             foreign_key: :order_id
 
+    # Status trails are stored as observations with concept 'Lab Order Status'
+    has_many :status_trail_observations,
+             lambda {
+               unscoped.where(voided: 0, concept_id: Lab::LabOrder.order_status_concept_id).order(obs_datetime: :asc)
+             },
+             class_name: 'Observation',
+             foreign_key: :order_id
+
     default_scope do
       joins(:order_type)
         .merge(OrderType.where(name: [
-          Lab::Metadata::ORDER_TYPE_NAME, 
-          Lab::Metadata::HTS_ORDER_TYPE_NAME
-        ]))
+                                 Lab::Metadata::ORDER_TYPE_NAME,
+                                 Lab::Metadata::HTS_ORDER_TYPE_NAME
+                               ]))
         .where.not(concept_id: ConceptName.where(name: 'Tests ordered').select(:concept_id))
     end
 
@@ -57,11 +70,13 @@ module Lab
     scope :not_drawn, -> { where(concept_id: ConceptName.where(name: 'Unknown').select(:concept_id)) }
 
     def self.prefetch_relationships
-      includes(:reason_for_test,
-               :requesting_clinician,
-               :target_lab,
-               :comment_to_fulfiller,
-               tests: [:result])
+      # NOTE: status_trail_observations and test results are not preloaded due to
+      # Rails limitations with eager loading unscoped associations. They load on-demand instead.
+      preload(:reason_for_test,
+              :requesting_clinician,
+              :target_lab,
+              :comment_to_fulfiller,
+              :tests)
     end
   end
 end

data/app/models/lab/lab_result.rb

@@ -3,9 +3,9 @@
 module Lab
   class LabResult < Observation
     def children
-      Observation.where(obs_group_id: obs_id, voided: 0)
+      Observation.unscoped.where(obs_group_id: obs_id, voided: 0)
     end
-    
+
     alias measures children
 
     default_scope do

data/app/models/lab/lab_test.rb

@@ -6,14 +6,33 @@ module Lab
       where(concept: ConceptName.where(name: Lab::Metadata::TEST_TYPE_CONCEPT_NAME))
     end
 
+    # Cache the concept IDs as class methods to avoid lookups in association scopes
+    def self.test_status_concept_id
+      @test_status_concept_id ||= ConceptName.find_by(name: 'Lab Test Status')&.concept_id
+    end
+
+    def self.test_result_concept_id
+      @test_result_concept_id ||= ConceptName.find_by(name: Lab::Metadata::TEST_RESULT_CONCEPT_NAME)&.concept_id
+    end
+
     has_one :result,
-            -> { where(concept: ConceptName.where(name: Lab::Metadata::TEST_RESULT_CONCEPT_NAME)) },
+            -> { unscoped.where(voided: 0, concept_id: Lab::LabTest.test_result_concept_id) },
             class_name: 'Lab::LabResult',
             foreign_key: :obs_group_id
 
+    # Status trails are stored as observations with concept 'Lab Test Status'
+    # They are linked via obs_group_id (this test obs is the parent)
+    has_many :status_trail_observations,
+             lambda {
+               unscoped.where(voided: 0, concept_id: Lab::LabTest.test_status_concept_id).order(obs_datetime: :asc)
+             },
+             class_name: 'Observation',
+             foreign_key: :obs_group_id,
+             primary_key: :obs_id
+
     def void(reason)
       result&.void(reason)
-      super(reason)
+      super
     end
   end
 end

data/app/serializers/lab/lab_order_serializer.rb

@@ -36,8 +36,10 @@ module Lab
             name: concept_name(reason_for_test&.value_coded)
           },
           delivery_mode: order&.lims_acknowledgement_status&.acknowledgement_type,
+          order_status: latest_order_status(order),
+          order_status_trail: serialize_order_status_trail(order),
           tests: tests.map do |test|
-            result_obs = test.children.first
+            result_obs = test.result
 
             {
               id: test.obs_id,
@@ -45,7 +47,9 @@ module Lab
               uuid: test.uuid,
               name: concept_name(test.value_coded),
               test_method: test_method(order, test.value_coded),
-              result: result_obs && ResultSerializer.serialize(result_obs)
+              result: result_obs && ResultSerializer.serialize(result_obs),
+              test_status: latest_test_status(test),
+              test_status_trail: serialize_test_status_trail(test)
             }
           end
         }
@@ -74,5 +78,72 @@ module Lab
                            .select(:concept_id)
       LabTest.unscoped.where(concept:, order:, voided: true)
     end
+
+    def self.latest_order_status(order)
+      # Query obs table for latest order status
+      latest_obs = order.status_trail_observations.last
+      return nil unless latest_obs
+
+      updated_by = parse_comments_json(latest_obs.comments)
+
+      {
+        status_id: 0, # status_id not used with text values
+        status: latest_obs.value_text,
+        timestamp: latest_obs.obs_datetime,
+        updated_by: updated_by
+      }
+    end
+
+    def self.serialize_order_status_trail(order)
+      # Query obs table for order status trail
+      order.status_trail_observations.map do |obs|
+        updated_by = parse_comments_json(obs.comments)
+
+        {
+          status_id: 0, # status_id not used with text values
+          status: obs.value_text,
+          timestamp: obs.obs_datetime,
+          updated_by: updated_by
+        }
+      end
+    end
+
+    def self.latest_test_status(test)
+      # Query obs table for latest test status
+      latest_obs = test.status_trail_observations.last
+      return nil unless latest_obs
+
+      updated_by = parse_comments_json(latest_obs.comments)
+
+      {
+        status_id: 0, # status_id not used with text values
+        status: latest_obs.value_text,
+        timestamp: latest_obs.obs_datetime,
+        updated_by: updated_by
+      }
+    end
+
+    def self.serialize_test_status_trail(test)
+      # Query obs table for test status trail
+      test.status_trail_observations.map do |obs|
+        updated_by = parse_comments_json(obs.comments)
+
+        {
+          status_id: 0, # status_id not used with text values
+          status: obs.value_text,
+          timestamp: obs.obs_datetime,
+          updated_by: updated_by
+        }
+      end
+    end
+
+    # Helper to parse updated_by from obs comments field
+    def self.parse_comments_json(comments)
+      return {} if comments.blank?
+
+      JSON.parse(comments)
+    rescue JSON::ParserError
+      {}
+    end
   end
 end

data/app/services/lab/concepts_service.rb

@@ -34,10 +34,11 @@ module Lab
 
       unless specimen_type
         return ActiveRecord::Base.connection.select_all <<~SQL
-          SELECT ca.concept_id, ca.value_reference as name, ca2.value_reference as nlims_code
+          SELECT ca.concept_id, ca.value_reference as name, ca2.value_reference as nlims_code, c.uuid
             FROM concept_attribute ca
           INNER JOIN concept_attribute ca2 ON ca.concept_id = ca2.concept_id
             AND ca2.attribute_type_id = #{ConceptAttributeType.nlims_code.concept_attribute_type_id}
+          INNER JOIN concept c ON c.concept_id = ca.concept_id
           WHERE ca.attribute_type_id = #{ConceptAttributeType.test_catalogue_name.concept_attribute_type_id}
           AND ca.concept_id IN (#{test_types.select(:concept_id).to_sql})
           GROUP BY ca.concept_id
@@ -56,10 +57,11 @@ module Lab
       )
 
       return ActiveRecord::Base.connection.select_all <<~SQL
-        SELECT ca.concept_id, ca.value_reference as name, ca2.value_reference as nlims_code
+        SELECT ca.concept_id, ca.value_reference as name, ca2.value_reference as nlims_code, c.uuid
           FROM concept_attribute ca
         INNER JOIN concept_attribute ca2 ON ca.concept_id = ca2.concept_id
           AND ca2.attribute_type_id = #{ConceptAttributeType.nlims_code.concept_attribute_type_id}
+        INNER JOIN concept c ON c.concept_id = ca.concept_id
         WHERE ca.attribute_type_id = #{ConceptAttributeType.test_catalogue_name.concept_attribute_type_id}
         AND ca.concept_id IN (#{concept_set.select(:concept_set).to_sql})
         GROUP BY ca.concept_id
@@ -72,10 +74,11 @@ module Lab
 
       unless test_type
         return ActiveRecord::Base.connection.select_all <<~SQL
-          SELECT ca.concept_id, ca.value_reference as name, ca2.value_reference as nlims_code
+          SELECT ca.concept_id, ca.value_reference as name, ca2.value_reference as nlims_code, c.uuid
             FROM concept_attribute ca
           INNER JOIN concept_attribute ca2 ON ca.concept_id = ca2.concept_id
             AND ca2.attribute_type_id = #{ConceptAttributeType.nlims_code.concept_attribute_type_id}
+          INNER JOIN concept c ON c.concept_id = ca.concept_id
           WHERE ca.attribute_type_id = #{ConceptAttributeType.test_catalogue_name.concept_attribute_type_id}
           AND ca.concept_id IN (#{specimen_types.select(:concept_id).to_sql})
           GROUP BY ca.concept_id
@@ -94,10 +97,11 @@ module Lab
       )
 
       return ActiveRecord::Base.connection.select_all <<~SQL
-        SELECT ca.concept_id, ca.value_reference as name, ca2.value_reference as nlims_code
+        SELECT ca.concept_id, ca.value_reference as name, ca2.value_reference as nlims_code, c.uuid
           FROM concept_attribute ca
         INNER JOIN concept_attribute ca2 ON ca.concept_id = ca2.concept_id
           AND ca2.attribute_type_id = #{ConceptAttributeType.nlims_code.concept_attribute_type_id}
+        INNER JOIN concept c ON c.concept_id = ca.concept_id
         WHERE ca.attribute_type_id = #{ConceptAttributeType.test_catalogue_name.concept_attribute_type_id}
         AND ca.concept_id IN (#{concept_set.pluck(:concept_id).push(0).join(',')})
         GROUP BY ca.concept_id
@@ -114,15 +118,16 @@ module Lab
       measures = ConceptSet.find_members_by_name(Lab::Metadata::TEST_RESULT_INDICATOR_CONCEPT_NAME)
                            .select(:concept_id)
 
-      sets = ConceptSet.where(concept_set: measures, concept_id: test)
+      sets = ConceptSet.where(concept_set: test, concept_id: measures)
 
       return ActiveRecord::Base.connection.select_all <<~SQL
-        SELECT ca.concept_id, ca.value_reference as name, ca2.value_reference as nlims_code
+        SELECT ca.concept_id, ca.value_reference as name, ca2.value_reference as nlims_code, c.uuid
           FROM concept_attribute ca
           INNER JOIN concept_attribute ca2 ON ca.concept_id = ca2.concept_id
             AND ca2.attribute_type_id = #{ConceptAttributeType.nlims_code.concept_attribute_type_id}
+          INNER JOIN concept c ON c.concept_id = ca.concept_id
           WHERE ca.attribute_type_id = #{ConceptAttributeType.test_catalogue_name.concept_attribute_type_id}
-          AND ca.concept_id IN (#{sets.pluck(:concept_set).push(0).join(',')})
+          AND ca.concept_id IN (#{sets.pluck(:concept_id).push(0).join(',')})
           GROUP BY ca.concept_id
       SQL
     end

data/app/services/lab/lims/api/rest_api.rb

@@ -77,15 +77,82 @@ module Lab
         def consume_orders(*_args, patient_id: nil, **_kwargs)
           orders_pending_updates(patient_id).each do |order|
             order_dto = Lab::Lims::OrderSerializer.serialize_order(order)
-            if order_dto['priority'].nil? || order_dto['sample_type'].casecmp?('not_specified')
-              patch_order_dto_with_lims_order!(order_dto, find_lims_order(order.accession_number))
+
+            # Always fetch the full order from NLIMS to get status trails
+            begin
+              lims_order = find_lims_order(order.accession_number)
+              patch_order_dto_with_lims_order!(order_dto, lims_order)
+
+              Rails.logger.debug("NLIMS order structure for #{order.accession_number}:")
+              Rails.logger.debug("  Has 'order' key: #{lims_order.key?('order')}")
+              Rails.logger.debug("  Has 'data' key: #{lims_order.key?('data')}")
+              Rails.logger.debug("  Top level keys: #{lims_order.keys.inspect}")
+
+              # Also extract status trails from the NLIMS order
+              # Note: NLIMS might return order data under 'order' or 'data.order'
+              order_data = lims_order['order'] || lims_order.dig('data', 'order') || lims_order
+
+              if order_data && order_data['status_trail']
+                Rails.logger.info("Found #{order_data['status_trail'].size} order status trail entries from NLIMS")
+                order_dto[:sample_statuses] ||= []
+                # Convert NLIMS status trail to the format expected by PullWorker
+                # Note: sample_statuses must be an array of single-key hashes
+                order_data['status_trail'].each do |trail|
+                  # Convert ISO 8601 timestamp to YYYYMMDDHHmmss format
+                  timestamp_key = convert_timestamp_to_key(trail['timestamp'])
+                  order_dto[:sample_statuses] << {
+                    timestamp_key => {
+                      'status_id' => trail['status_id'],
+                      'status' => trail['status'],
+                      'updated_by' => trail['updated_by']
+                    }
+                  }
+                  Rails.logger.debug("  Added order status: #{trail['status']} at #{timestamp_key}")
+                end
+                Rails.logger.debug("Final sample_statuses: #{order_dto[:sample_statuses].inspect}")
+              else
+                Rails.logger.warn("No order status_trail found in NLIMS response for #{order.accession_number}")
+                Rails.logger.debug("Order data keys: #{order_data&.keys&.inspect}")
+              end
+
+              # Extract test status trails from NLIMS tests
+              tests_data = lims_order['tests'] || lims_order.dig('data', 'tests') || []
+              if tests_data.is_a?(Array)
+                Rails.logger.debug("Processing #{tests_data.size} tests from NLIMS")
+                order_dto['test_statuses'] ||= {}
+                tests_data.each do |test|
+                  next unless test['status_trail'].is_a?(Array)
+
+                  test_name = test.dig('test_type', 'name')
+                  next unless test_name
+
+                  Rails.logger.debug("  Found #{test['status_trail'].size} status trail entries for test #{test_name}")
+                  order_dto['test_statuses'][test_name] ||= {}
+                  test['status_trail'].each do |trail|
+                    # Convert ISO 8601 timestamp to YYYYMMDDHHmmss format
+                    timestamp_key = convert_timestamp_to_key(trail['timestamp'])
+                    order_dto['test_statuses'][test_name][timestamp_key] = {
+                      'status_id' => trail['status_id'],
+                      'status' => trail['status'],
+                      'updated_by' => trail['updated_by']
+                    }
+                  end
+                end
+              end
+            rescue RestClient::NotFound
+              Rails.logger.warn("Order ##{order.accession_number} not found in NLIMS, using local data only")
             end
+
+            # Try to fetch results if available
             if order_dto['test_results'].empty?
               begin
                 patch_order_dto_with_lims_results!(order_dto, find_lims_results(order.accession_number))
-              rescue InvalidParameters => e # LIMS responds with a 401 when a result is not found :(
-                Rails.logger.error("Failed to fetch results for ##{order.accession_number}: #{e.message}")
-                next
+              rescue InvalidParameters => e
+                Rails.logger.info("No results available for ##{order.accession_number}: #{e.message}")
+                # Don't skip - continue processing to save status trails
+              rescue RestClient::NotFound
+                Rails.logger.info("No results found for ##{order.accession_number}")
+                # Don't skip - continue processing to save status trails
               end
             end
 
@@ -550,6 +617,20 @@ module Lab
 
           orders
         end
+
+        # Converts ISO 8601 timestamp to YYYYMMDDHHmmss format
+        def convert_timestamp_to_key(timestamp)
+          return timestamp if timestamp.nil? || timestamp.empty?
+
+          begin
+            # Parse ISO 8601 timestamp and format as YYYYMMDDHHmmss
+            Time.parse(timestamp).strftime('%Y%m%d%H%M%S')
+          rescue StandardError => e
+            Rails.logger.warn("Failed to parse timestamp '#{timestamp}': #{e.message}")
+            # Fallback: remove all non-digits
+            timestamp.to_s.gsub(/\D/, '')
+          end
+        end
       end
     end
   end

data/app/services/lab/lims/pull_worker.rb

@@ -191,6 +191,11 @@ module Lab
       def create_order(patient, order_dto)
         logger.debug("Creating order ##{order_dto['_id']}")
         order = OrdersService.order_test(order_dto.to_order_service_params(patient_id: patient.patient_id))
+
+        # Extract and save status trails from NLIMS
+        save_status_trails_from_nlims(order, order_dto)
+
+        # Update results if present
         update_results(order, order_dto['test_results']) unless order_dto['test_results'].empty?
 
         order
@@ -200,6 +205,11 @@ module Lab
         logger.debug("Updating order ##{order_dto['_id']}")
         order = OrdersService.update_order(order_id, order_dto.to_order_service_params(patient_id: patient.patient_id)
                                                               .merge(force_update: 'true'))
+
+        # Extract and save status trails from NLIMS
+        save_status_trails_from_nlims(order, order_dto)
+
+        # Update results if present
         update_results(order, order_dto['test_results']) unless order_dto['test_results'].empty?
 
         order
@@ -290,6 +300,165 @@ module Lab
         "#{id}:#{first_name} #{last_name}:#{phone_number}"
       end
 
+      def save_status_trails_from_nlims(order, order_dto)
+        logger.debug("Saving status trails from NLIMS for order ##{order['order_id'] || order[:order_id]}")
+        logger.debug("Order DTO keys: #{order_dto.keys.inspect}")
+        logger.debug("Order DTO sample_statuses type: #{order_dto[:sample_statuses].class}")
+        logger.debug("Order DTO sample_statuses content: #{order_dto[:sample_statuses].inspect}")
+
+        # Extract order status trail from sample_statuses (NLIMS uses sample_statuses for order status)
+        # Note: sample_statuses is an array of single-key hashes
+        if order_dto[:sample_statuses].is_a?(Array)
+          logger.debug("Found sample_statuses: #{order_dto[:sample_statuses].size} entries")
+          save_order_status_trails(order, order_dto[:sample_statuses])
+        else
+          logger.warn("No sample_statuses found or not an Array: #{order_dto[:sample_statuses].class}")
+        end
+
+        # Extract test status trails from test_statuses
+        if order_dto['test_statuses'].is_a?(Hash)
+          logger.debug("Found test_statuses: #{order_dto['test_statuses'].keys.size} entries")
+          save_test_status_trails(order, order_dto['test_statuses'])
+        else
+          logger.debug("No test_statuses found or not a Hash: #{order_dto['test_statuses'].class}")
+        end
+      end
+
+      def save_order_status_trails(order, sample_statuses)
+        logger.debug("Saving order status trails for order ##{order['order_id'] || order[:order_id]}")
+        logger.debug("Sample statuses: #{sample_statuses.inspect}")
+
+        # Find concept
+        order_status_concept = ConceptName.find_by(name: 'Lab Order Status')&.concept
+        unless order_status_concept
+          logger.error('Lab Order Status concept not found')
+          return
+        end
+
+        order_id = order['order_id'] || order[:order_id] || order['id'] || order[:id]
+        lab_order = Lab::LabOrder.find_by(order_id: order_id)
+        unless lab_order
+          logger.error("Order not found: #{order_id}")
+          return
+        end
+
+        # sample_statuses is an array of single-key hashes like:
+        # [{ "20260225120000" => { "status" => "Drawn", ... } }, { "20260225130000" => { ... } }]
+        sample_statuses.each do |trail_entry|
+          # Each trail_entry is a hash with one timestamp key
+          trail_entry.each_pair do |timestamp_key, status_data|
+            next unless status_data.is_a?(Hash) && status_data['status']
+
+            # Parse timestamp (format: YYYYMMDDHHmmss) - already in local timezone from NLIMS conversion
+            # Use Time.zone.strptime to prevent Rails from converting timezone during save
+            begin
+              timestamp = Time.zone.strptime(timestamp_key, '%Y%m%d%H%M%S')
+            rescue StandardError => e
+              logger.warn("Failed to parse timestamp '#{timestamp_key}': #{e.message}")
+              next
+            end
+
+            # Skip if this status has already been recorded for this order (regardless of timestamp)
+            if Observation.unscoped.exists?(
+              person_id: lab_order.patient_id,
+              order_id: order_id,
+              concept_id: order_status_concept.concept_id,
+              value_text: status_data['status'],
+              voided: 0
+            )
+              logger.debug("Order status already recorded: #{status_data['status']} for order ##{order_id}")
+              next
+            end
+
+            updated_by = status_data['updated_by'] || {}
+
+            begin
+              Observation.create!(
+                person_id: lab_order.patient_id,
+                encounter_id: lab_order.encounter_id,
+                order_id: order_id,
+                concept_id: order_status_concept.concept_id,
+                value_text: status_data['status'], # Store status as text
+                obs_datetime: timestamp,
+                comments: updated_by.to_json,
+                creator: User.current&.user_id || 1,
+                date_created: Time.now,
+                uuid: SecureRandom.uuid
+              )
+              logger.info("Created order status trail: #{status_data['status']} at #{timestamp}")
+            rescue StandardError => e
+              logger.error("Failed to save order status trail: #{e.message}")
+              logger.error("  Order ID: #{order_id}")
+              logger.error("  Status: #{status_data['status']}")
+              logger.error("  Timestamp: #{timestamp}")
+            end
+          end
+        end
+      end
+
+      def save_test_status_trails(order, test_statuses)
+        # Find test status concept
+        test_status_concept = ConceptName.find_by(name: 'Lab Test Status')&.concept
+        unless test_status_concept
+          logger.error('Lab Test Status concept not found')
+          return
+        end
+
+        test_statuses.each do |test_name, statuses|
+          next unless statuses.is_a?(Hash)
+
+          # Find the test by name
+          test_concept = Utils.find_concept_by_name(Utils.translate_test_name(test_name))
+          next unless test_concept
+
+          test = Lab::LabTest.find_by(order_id: order['order_id'], value_coded: test_concept.concept_id)
+          next unless test
+
+          # Process each status in the trail
+          statuses.each do |timestamp_key, status_data|
+            next unless status_data.is_a?(Hash) && status_data['status']
+
+            # Parse timestamp (format: YYYYMMDDHHmmss) - already in local timezone from NLIMS conversion
+            # Use Time.zone.strptime to prevent Rails from converting timezone during save
+            begin
+              timestamp = Time.zone.strptime(timestamp_key, '%Y%m%d%H%M%S')
+            rescue StandardError => e
+              logger.warn("Failed to parse timestamp '#{timestamp_key}': #{e.message}")
+              next
+            end
+
+            # Skip if this status has already been recorded for this test (regardless of timestamp)
+            next if Observation.unscoped.exists?(
+              person_id: test.person_id,
+              obs_group_id: test.obs_id,
+              concept_id: test_status_concept.concept_id,
+              value_text: status_data['status'],
+              voided: 0
+            )
+
+            updated_by = status_data['updated_by'] || {}
+
+            begin
+              Observation.create!(
+                person_id: test.person_id,
+                encounter_id: test.encounter_id,
+                obs_group_id: test.obs_id,
+                concept_id: test_status_concept.concept_id,
+                value_text: status_data['status'], # Store status as text
+                obs_datetime: timestamp,
+                comments: updated_by.to_json,
+                creator: User.current&.user_id || 1,
+                date_created: Time.now,
+                uuid: SecureRandom.uuid
+              )
+              logger.info("Created test status trail: #{status_data['status']} at #{timestamp} for #{test_name}")
+            rescue StandardError => e
+              logger.error("Failed to save test status trail for #{test_name}: #{e.message}")
+            end
+          end
+        end
+      end
+
       def save_failed_import(order_dto, reason, diff = nil)
         logger.info("Failed to import LIMS order ##{order_dto[:tracking_number]} due to '#{reason}'")
         LimsFailedImport.create!(lims_id: order_dto[:_id],

data/app/services/lab/metadata.rb

@@ -12,6 +12,7 @@ module Lab
     TEST_RESULT_INDICATOR_CONCEPT_NAME = 'Lab test result indicator'
     TEST_TYPE_CONCEPT_NAME = 'Test type'
     LAB_ORDER_STATUS_CONCEPT_NAME = 'lab order status'
+    LAB_TEST_STATUS_CONCEPT_NAME = 'lab test status'
     UNKNOWN_SPECIMEN = 'Unknown'
     VISIT_TYPE_UUID = '8a4a4488-c2cc-11de-8d13-0010c6dffd0f'
     TEST_METHOD_CONCEPT_NAME = 'Recommended test method'

data/app/services/lab/orders_service.rb

@@ -60,8 +60,14 @@ module Lab
 
           attach_test_method(order, order_params) if order_params[:test_method]
 
+          # Create initial order status trail
+          create_initial_order_status_trail(order)
+
           Lab::TestsService.create_tests(order, order_params[:date], order_params[:tests])
 
+          # Reload order to include status trails and tests
+          order = Lab::LabOrder.prefetch_relationships.find(order.order_id)
+
           Lab::LabOrderSerializer.serialize_order(
             order, requesting_clinician: add_requesting_clinician(order, order_params),
                    reason_for_test: add_reason_for_test(order, order_params),
@@ -102,14 +108,16 @@ module Lab
 
         if reason_for_test
           Rails.logger.debug("Updating reason for test on order ##{order.order_id}")
-          update_reason_for_test(order, Concept.find(reason_for_test)&.id, force_update: params.fetch('force_update', false))
+          update_reason_for_test(order, Concept.find(reason_for_test)&.id,
+                                 force_update: params.fetch('force_update', false))
         end
 
         Lab::LabOrderSerializer.serialize_order(order)
       end
 
       def void_order(order_id, reason)
-        order = Lab::LabOrder.includes(%i[requesting_clinician reason_for_test target_lab comment_to_fulfiller], tests: [:result])
+        order = Lab::LabOrder.includes(%i[requesting_clinician reason_for_test target_lab comment_to_fulfiller],
+                                       tests: [:result])
                              .find(order_id)
 
         order.requesting_clinician&.void(reason)
@@ -140,14 +148,55 @@ module Lab
             value_text: order_params['status'],
             creator: User.current.id
           )
+
+          # Save order status trail if available
+          save_order_status_trail(order, order_params) if order_params['status']
         end
         create_rejection_notification(order_params) if order_params['status'] == 'test-rejected'
       end
 
       def update_order_result(order_params)
-        order = find_order(order_params['tracking_number'])
+        # Extract tracking number from nested structure if present
+        tracking_number = order_params['tracking_number'] || order_params.dig('order', 'tracking_number')
+        order = find_order(tracking_number)
+
         order_dto = Lab::Lims::OrderSerializer.serialize_order(order)
-        patch_order_dto_with_lims_results!(order_dto, order_params['results'])
+
+        # Handle results if present in the old format
+        patch_order_dto_with_lims_results!(order_dto, order_params['results']) if order_params['results']
+
+        # Handle test results in NLIMS format
+        if order_params['tests']
+          # Extract test results from NLIMS tests payload
+          test_results = {}
+          order_params['tests'].each do |test_data|
+            test_name = test_data.dig('test_type', 'name')
+            next unless test_name && test_data['test_results']
+
+            test_results[test_name] = {
+              'results' => test_data['test_results'].each_with_object({}) do |result, formatted|
+                measure_name = result.dig('measure', 'name')
+                result_value = result.dig('result', 'value')
+                next unless measure_name && result_value
+
+                formatted[measure_name] = { 'result_value' => result_value }
+              end,
+              'result_date' => test_data['test_results'].first&.dig('result', 'result_date'),
+              'result_entered_by' => {}
+            }
+          end
+
+          patch_order_dto_with_lims_results!(order_dto, test_results) unless test_results.empty?
+        end
+
+        # Save order status trail if available from NLIMS
+        if order_params['order'] && order_params['order']['status_trail']
+          save_order_status_trails_from_nlims(order, order_params['order']['status_trail'])
+        end
+
+        # Save test status trails if available from NLIMS
+        save_test_status_trails_from_nlims(order, order_params['tests']) if order_params['tests']
+
         Lab::Lims::PullWorker.new(nil).process_order(order_dto)
       end
 
@@ -160,13 +209,15 @@ module Lab
           last_order_date: Lab::LabOrder.last&.start_date&.to_date,
           lab_orders: []
         }
-        data[:lab_orders] = orders.map do |order|
-          Lab::LabOrderSerializer.serialize_order(
-            order, requesting_clinician: order.requesting_clinician,
-                   reason_for_test: order.reason_for_test,
-                   target_lab: order.target_lab
-          )
-        end if include_data
+        if include_data
+          data[:lab_orders] = orders.map do |order|
+            Lab::LabOrderSerializer.serialize_order(
+              order, requesting_clinician: order.requesting_clinician,
+                     reason_for_test: order.reason_for_test,
+                     target_lab: order.target_lab
+            )
+          end
+        end
         data
       end
 
@@ -174,14 +225,14 @@ module Lab
 
       def create_rejection_notification(order_params)
         order = find_order order_params['tracking_number']
-        data = { 'type': 'LIMS',
-                 'specimen': ConceptName.find_by(concept_id: order.concept_id)&.name,
-                 'accession_number': order&.accession_number,
-                 'order_date': order&.start_date,
-                 'arv_number': find_arv_number(order.patient_id),
-                 'patient_id': result.person_id,
-                 'ordered_by': order&.provider&.person&.name,
-                 'rejection_reason': order_params['comments'] }.as_json
+        data = { type: 'LIMS',
+                 specimen: ConceptName.find_by(concept_id: order.concept_id)&.name,
+                 accession_number: order&.accession_number,
+                 order_date: order&.start_date,
+                 arv_number: find_arv_number(order.patient_id),
+                 patient_id: result.person_id,
+                 ordered_by: order&.provider&.person&.name,
+                 rejection_reason: order_params['comments'] }.as_json
         NotificationService.new.create_notification('LIMS', data)
       end
 
@@ -235,7 +286,9 @@ module Lab
         encounter.encounter_datetime = order_params[:date] || Date.today
         encounter.visit = Visit.find_by_uuid(visit) if Encounter.column_names.include?('visit_id')
         encounter.provider_id = User.current&.person&.id if Encounter.column_names.include?('provider_id')
-        encounter.program_id = order_params[:program_id] if Encounter.column_names.include?('program_id') && order_params[:program_id].present?
+        if Encounter.column_names.include?('program_id') && order_params[:program_id].present?
+          encounter.program_id = order_params[:program_id]
+        end
         encounter.save!
         encounter.reload
       end
@@ -247,7 +300,6 @@ module Lab
 
         concept = params.dig(:specimen, :concept)
         concept ||= params.dig(:specimen, :concept_id)
-        concept ||= unknown_concept_id
 
         order_type = nil
         order_type = OrderType.find_by_order_type_id!(params[:order_type_id])&.id if params[:order_type_id].present?
@@ -260,7 +312,8 @@ module Lab
         order.date_created = params[:date]&.to_date || Date.today if order.respond_to?(:date_created)
         order.start_date = params[:date]&.to_date || Date.today if order.respond_to?(:start_date)
         order.auto_expire_date = params[:end_date]
-        order.comment_to_fulfiller = params[:comment_to_fulfiller] if params[:comment_to_fulfiller]
+        # NOTE: comment_to_fulfiller is a has_one association, not a field
+        # It will be created via add_comment_to_fulfiller method
         order.accession_number = access_number
         order.orderer = User.current&.user_id
 
@@ -348,7 +401,7 @@ module Lab
       end
 
       def unknown_concept_id
-        ConceptName.find_by_name!('Unknown').concept_id
+        ConceptName.find_by_name!('Unknown').concept
       end
 
       def update_reason_for_test(order, concept_id, force_update: false)
@@ -356,7 +409,10 @@ module Lab
 
         return if order.reason_for_test&.value_coded == concept_id
 
-        raise InvalidParameterError, "Can't change reason for test once set" if order.reason_for_test&.value_coded && !force_update
+        if order.reason_for_test&.value_coded && !force_update
+          raise InvalidParameterError,
+                "Can't change reason for test once set"
+        end
 
         order.reason_for_test&.delete
         date = order.start_date if order.respond_to?(:start_date)
@@ -369,6 +425,150 @@ module Lab
           obs.void('New Status Received from LIMS')
         end
       end
+
+      def create_initial_order_status_trail(order)
+        create_order_status_observation(
+          order: order,
+          status: 'Drawn',
+          timestamp: order.start_date || order.date_created || Time.now,
+          updated_by: {
+            'first_name' => User.current&.person&.names&.first&.given_name,
+            'last_name' => User.current&.person&.names&.first&.family_name,
+            'id' => User.current&.user_id&.to_s,
+            'phone_number' => nil
+          }
+        )
+      rescue StandardError => e
+        Rails.logger.warn("Failed to create initial order status trail: #{e.message}")
+      end
+
+      def save_order_status_trail(order, status_params)
+        create_order_status_observation(
+          order: order,
+          status: status_params['status'],
+          timestamp: status_params['status_time'] || Time.now,
+          updated_by: status_params['updated_by'] || {}
+        )
+      end
+
+      def save_order_status_trails_from_nlims(order, status_trail)
+        return unless status_trail.is_a?(Array)
+
+        status_trail.each do |trail_entry|
+          create_order_status_observation(
+            order: order,
+            status: trail_entry['status'],
+            timestamp: trail_entry['timestamp'],
+            updated_by: trail_entry['updated_by'] || {}
+          )
+        end
+      end
+
+      def save_test_status_trails_from_nlims(order, tests)
+        return unless tests.is_a?(Array)
+
+        tests.each do |test_data|
+          next unless test_data['status_trail'].is_a?(Array)
+
+          # Find the test by test_type
+          test_name = test_data.dig('test_type', 'name')
+          next unless test_name
+
+          # Find concept for this test
+          concept = Lab::Lims::Utils.find_concept_by_name(test_name)
+          next unless concept
+
+          # Find the test observation
+          test = order.tests.find_by(value_coded: concept.concept_id)
+          next unless test
+
+          # Save each status trail entry
+          test_data['status_trail'].each do |trail_entry|
+            create_test_status_observation(
+              test: test,
+              status: trail_entry['status'],
+              timestamp: trail_entry['timestamp'],
+              updated_by: trail_entry['updated_by'] || {}
+            )
+          end
+        end
+      end
+
+      # Creates an observation for order status trail using obs table
+      def create_order_status_observation(order:, status:, timestamp:, updated_by: {})
+        # Find concept
+        order_status_concept = ConceptName.find_by(name: 'Lab Order Status')&.concept
+
+        unless order_status_concept
+          Rails.logger.warn('Missing Lab Order Status concept')
+          return
+        end
+
+        # Check if this exact status already exists
+        return if Observation.unscoped.exists?(
+          person_id: order.patient_id,
+          order_id: order.order_id,
+          concept_id: order_status_concept.concept_id,
+          value_text: status,
+          obs_datetime: timestamp,
+          voided: 0
+        )
+
+        # Create status observation
+        Observation.create!(
+          person_id: order.patient_id,
+          encounter_id: order.encounter_id,
+          order_id: order.order_id,
+          concept_id: order_status_concept.concept_id,
+          value_text: status, # Store status as text
+          obs_datetime: timestamp,
+          comments: updated_by.to_json,
+          creator: User.current&.user_id || 1,
+          date_created: Time.now,
+          uuid: SecureRandom.uuid
+        )
+      rescue StandardError => e
+        Rails.logger.error("Failed to create order status observation: #{e.message}")
+        Rails.logger.error(e.backtrace.first(5).join("\n"))
+      end
+
+      # Creates an observation for test status trail using obs table
+      def create_test_status_observation(test:, status:, timestamp:, updated_by: {})
+        # Find concept
+        test_status_concept = ConceptName.find_by(name: 'Lab Test Status')&.concept
+
+        unless test_status_concept
+          Rails.logger.warn('Missing Lab Test Status concept')
+          return
+        end
+
+        # Check if this exact status already exists
+        return if Observation.unscoped.exists?(
+          person_id: test.person_id,
+          obs_group_id: test.obs_id,
+          concept_id: test_status_concept.concept_id,
+          value_text: status,
+          obs_datetime: timestamp,
+          voided: 0
+        )
+
+        # Create status observation
+        Observation.create!(
+          person_id: test.person_id,
+          encounter_id: test.encounter_id,
+          obs_group_id: test.obs_id, # Link to parent test observation
+          concept_id: test_status_concept.concept_id,
+          value_text: status, # Store status as text
+          obs_datetime: timestamp,
+          comments: updated_by.to_json,
+          creator: User.current&.user_id || 1,
+          date_created: Time.now,
+          uuid: SecureRandom.uuid
+        )
+      rescue StandardError => e
+        Rails.logger.error("Failed to create test status observation: #{e.message}")
+        Rails.logger.error(e.backtrace.first(5).join("\n"))
+      end
     end
   end
 end

data/app/services/lab/tests_service.rb

@@ -14,8 +14,7 @@ module Lab
 
         tests = filter_tests(tests, test_type_id: filters.delete(:test_type_id),
                                     patient_id: filters.delete(:patient_id),
-                                    patient: filters.delete(:patient)
-                                    )
+                                    patient: filters.delete(:patient))
 
         tests = filter_tests_by_results(tests) if %w[1 true].include?(filters[:pending_results]&.downcase)
 
@@ -29,12 +28,11 @@ module Lab
       def create_tests(order, date, tests_params)
         raise InvalidParameterError, 'tests are required' if tests_params.nil? || tests_params.empty?
 
-        
         Lab::LabTest.transaction do
           tests_params.map do |params|
             concept_id = params[:concept_id]
             concept_id = Concept.find_concept_by_uuid(params[:concept]).id if concept_id.nil?
-            
+
             test = Lab::LabTest.create!(
               concept_id: ConceptName.find_by_name!(Lab::Metadata::TEST_TYPE_CONCEPT_NAME)
                                      .concept_id,
@@ -45,6 +43,9 @@ module Lab
               value_coded: concept_id
             )
 
+            # Create initial test status trail
+            create_initial_test_status_trail(test, date)
+
             Lab::TestSerializer.serialize(test, order:)
           end
         end
@@ -100,6 +101,49 @@ module Lab
           value_coded: test_type_id
         )
       end
+
+      def create_initial_test_status_trail(test, date)
+        # Find concept
+        test_status_concept = ConceptName.find_by(name: 'Lab Test Status')&.concept
+
+        unless test_status_concept
+          Rails.logger.warn('Missing Lab Test Status concept')
+          return
+        end
+
+        timestamp = date || test.obs_datetime || Time.now
+
+        # Check if 'Drawn' status already exists for this test
+        return if Observation.unscoped.exists?(
+          person_id: test.person_id,
+          obs_group_id: test.obs_id,
+          concept_id: test_status_concept.concept_id,
+          value_text: 'Drawn',
+          voided: 0
+        )
+
+        # Create status observation with 'Drawn' as initial status
+        Observation.create!(
+          person_id: test.person_id,
+          encounter_id: test.encounter_id,
+          obs_group_id: test.obs_id, # Link to parent test observation
+          concept_id: test_status_concept.concept_id,
+          value_text: 'Drawn', # Store status as text
+          obs_datetime: timestamp,
+          status: 'FINAL',
+          comments: {
+            'first_name' => User.current&.person&.names&.first&.given_name,
+            'last_name' => User.current&.person&.names&.first&.family_name,
+            'id' => User.current&.user_id&.to_s,
+            'phone_number' => nil
+          }.to_json,
+          creator: User.current&.user_id || 1,
+          date_created: Time.now,
+          uuid: SecureRandom.uuid
+        )
+      rescue StandardError => e
+        Rails.logger.warn("Failed to create initial test status trail: #{e.message}")
+      end
     end
   end
 end

data/db/migrate/20260226065149_create_lab_status_concepts.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+# This migration creates concepts for lab order and test status tracking using obs table
+# Status values are stored as text (value_text), not coded values
+class CreateLabStatusConcepts < ActiveRecord::Migration[5.2]
+  def up
+    ActiveRecord::Base.transaction do
+      # Find concept class and datatype
+      concept_class = ConceptClass.find_by(name: 'Finding') || ConceptClass.find_by(name: 'Misc')
+      text_datatype = ConceptDatatype.find_by(name: 'Text')
+
+      # Create Lab Order Status concept (stores status as value_text)
+      unless ConceptName.exists?(name: 'Lab Order Status')
+        order_status_concept = create_concept(
+          name: 'Lab Order Status',
+          class_id: concept_class.id,
+          datatype_id: text_datatype.id,
+          is_set: 0
+        )
+        puts "Created 'Lab Order Status' concept with ID: #{order_status_concept.concept_id}"
+      end
+
+      # Create Lab Test Status concept (stores status as value_text)
+      unless ConceptName.exists?(name: 'Lab Test Status')
+        test_status_concept = create_concept(
+          name: 'Lab Test Status',
+          class_id: concept_class.id,
+          datatype_id: text_datatype.id,
+          is_set: 0
+        )
+        puts "Created 'Lab Test Status' concept with ID: #{test_status_concept.concept_id}"
+      end
+
+      puts 'Lab status concepts created successfully'
+    end
+  end
+
+  def down
+    ActiveRecord::Base.transaction do
+      # Remove concepts
+      concepts_to_remove = ['Lab Order Status', 'Lab Test Status']
+
+      concepts_to_remove.each do |name|
+        concept = ConceptName.find_by(name: name)&.concept
+        next unless concept
+
+        ConceptName.where(concept: concept).destroy_all
+        concept.destroy
+      end
+    end
+  end
+
+  private
+
+  def create_concept(name:, class_id:, datatype_id:, is_set:)
+    concept = Concept.create!(
+      class_id: class_id,
+      datatype_id: datatype_id,
+      short_name: name,
+      retired: 0,
+      is_set: is_set,
+      creator: 1,
+      date_created: Time.current,
+      uuid: SecureRandom.uuid
+    )
+
+    ConceptName.create!(
+      concept: concept,
+      name: name,
+      locale: 'en',
+      locale_preferred: 1,
+      concept_name_type: 'FULLY_SPECIFIED',
+      creator: 1,
+      date_created: Time.current,
+      uuid: SecureRandom.uuid
+    )
+
+    concept
+  end
+end

data/lib/lab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lab
-  VERSION = '2.1.8'
+  VERSION = '2.1.9-alpha'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: his_emr_api_lab
 version: !ruby/object:Gem::Version
-  version: 2.1.8
+  version: 2.1.9.pre.alpha
 platform: ruby
 authors:
 - Elizabeth Glaser Pediatric Foundation Malawi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-02-27 00:00:00.000000000 Z
+date: 2026-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: couchrest
@@ -302,6 +302,7 @@ files:
 - db/migrate/20260119104240_add_fulfiller_fields_to_orders.rb
 - db/migrate/20260119104241_create_comment_to_fulfiller_concept.rb
 - db/migrate/20260128111557_add_program_id_to_encounter.rb
+- db/migrate/20260226065149_create_lab_status_concepts.rb
 - lib/auto12epl.rb
 - lib/couch_bum/couch_bum.rb
 - lib/generators/lab/install/USAGE
@@ -340,9 +341,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.4.1
 signing_key: