malawi_hiv_program_reports 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c6bffbef9d85fec441ab368a631e8e730446a87ed8722d2a60d348c68ecf969
-  data.tar.gz: dfd3ba0bfe3a2fc65ca948d3dd2bee21c9f8bb4f7c42ade37623f46c6fb8aefb
+  metadata.gz: af23c66f80f04922477b707fe4978ca1234e06814761e657a4e9a27c8bb433e3
+  data.tar.gz: 762826dfec0aafbf978fc024ab7dafca271d3d6588128c9d4a9180a0ffcf86ab
 SHA512:
-  metadata.gz: 9a0217e328962a1734f08e22d28df28624f0a6eff6ca02e1eb32f021e0397758ab58bca452ddada4ccf662cac36b862952c880b98e3c08e500e4160db51af968
-  data.tar.gz: c6f54d2927268c90cf1e5b87ea621eb68aa5cea3d6f17095f382f7c7f91bdd0f4360a9ab4e00f8be536414a498c2dd1b3d01d8a7f88e01425dbb0e0292309a1c
+  metadata.gz: dad18495c08f5e2ca3db34adad758449cdf147dbd8c9edabbb8b771f68e0dda3d4ae88d47082a24fa5ef62f52e74f603676f8bef15d896ae93947ba529421132
+  data.tar.gz: e3497d0e34c8397efec79d71d6e77e805b3d580e1b8096856a04a2e97d6fe7da73612bf593a58c680b13b93c73778f014d61ea3edfb5ba7fe690e31f62fbc778

data/app/services/malawi_hiv_program_reports/moh/cumulative_cohort.rb

@@ -25,14 +25,15 @@ module MalawiHivProgramReports
 
       def find_report
         start_time = Time.now
-        prepare_tables
-        clear_tables if rebuild
+        handle_tables
         clear_cohort_status unless rebuild
-        process_thread
+        process_thread(locations:)
         end_time = Time.now
         time_in_minutes = ((end_time - start_time) / 60).round(2)
+        handle_failed
+        time_taken_for_failed = ((Time.now - end_time) / 60).round(2)
         Rails.logger.info("Cumulative Cohort report took #{time_in_minutes} minutes to generate for these locations: #{locations}")
-        { cohort_time: time_in_minutes }
+        { cohort_time: time_in_minutes, processing_failed: time_taken_for_failed }
       end
 
       private
@@ -46,7 +47,7 @@ module MalawiHivProgramReports
       # 6. Finally we want to get the outcomes of the clients in 5 above
 
       # rubocop:disable Metrics/MethodLength
-      def process_thread
+      def process_thread(locations: [])
         # use locations to thread but process 10 locations at a time
         queue = Queue.new
         locations.each { |loc| queue << loc }
@@ -76,6 +77,51 @@ module MalawiHivProgramReports
       end
       # rubocop:enable Metrics/MethodLength
 
+      def reprocess_failed
+        failed_locs = failed_locs&.map { |loc| loc[:site_id] }
+        return if failed_locs.empty?
+
+        process_thread(locations: failed_locs)
+      end
+
+      def handle_failed
+        failed_locs = failed_sites&.map { |loc| loc['site_id'].to_i }
+        return if failed_locs.empty?
+
+        4.times do
+          reprocess_failed
+        end
+
+        failed_locs = failed_sites&.map { |loc| loc['site_id'].to_i }
+        return if failed_locs.empty?
+
+        failed_locs.each do |loc|
+          ActiveRecord::Base.connection_pool.with_connection do
+            process_data loc
+          rescue StandardError => e
+            Rails.logger.info("Error processing location #{loc}: #{e.message}")
+            Rails.logger.info(e.backtrace.join("\n"))
+            save_incomplete_site(location: loc, time_taken: 0)
+          end
+        end
+      end
+
+      def failed_sites
+        ActiveRecord::Base.connection.select_all <<~SQL
+          SELECT site_id
+          FROM cdr_temp_cohort_status
+          WHERE status = 'incomplete' AND site_id IN (#{locations.join(',')})
+        SQL
+      end
+
+      def handle_tables
+        prepare_tables
+        clear_tables if rebuild
+        outcome = MalawiHivProgramReports::Moh::CumulativeOutcome.new(end_date:, location: 0, definition:, rebuild:,
+                                                                      start_date:)
+        outcome.handle_tables
+      end
+
       def process_data(location)
         start_time = Time.now
         if rebuild

data/app/services/malawi_hiv_program_reports/moh/cumulative_outcome.rb

@@ -4,7 +4,6 @@ module MalawiHivProgramReports
   module Moh
     # This is the Cumulative Cohort Builder class
     # rubocop:disable Metrics/ClassLength
-    # rubocop:disable Metrics/MethodLength
     class CumulativeOutcome
       include MalawiHivProgramReports::Utils::CommonSqlQueryUtils
       attr_reader :end_date, :definition, :rebuild, :start_date, :prev_date, :location
@@ -17,11 +16,9 @@ module MalawiHivProgramReports
         @definition = definition
         @rebuild = kwargs[:rebuild]
         @location = kwargs[:location]
-        Rails.logger.info "Definition: #{definition}. Rebuild: #{@rebuild}. Location: #{@location}. Start date: #{@start_date}. End date: #{@end_date}. Previous date: #{@prev_date}."
       end
 
       def find_report
-        prepare_tables
         [false, true].each do |start|
           clear_tables(start:) if rebuild
           update_steps(start:, portion: false) unless rebuild
@@ -30,15 +27,18 @@ module MalawiHivProgramReports
       end
 
       def update_outcomes_by_definition
-        prepare_tables
         [false, true].each do |start|
-          update_steps(start: start, portion: true)
+          update_steps(start:, portion: true)
           load_patients_on_treatment(start:)
           load_without_clinical_contact(start:)
           load_defaulters(start:)
         end
       end
 
+      def handle_tables
+        prepare_tables
+      end
+
       private
 
       # The main idea here is to come up with cumulative outcomes for patients in temp_earliest_start_date
@@ -61,7 +61,6 @@ module MalawiHivProgramReports
       # ===================================
       #  Data Management Region
       # ===================================
-      # rubocop:disable Metrics/MethodLength
       def process_data(start: false)
         denormalize(start:)
         # HIC SUNT DRACONIS: The order of the operations below matters,
@@ -95,7 +94,7 @@ module MalawiHivProgramReports
             AND drug_order.site_id = o.site_id AND drug_order.quantity > 0
             AND drug_order.drug_inventory_id IN (#{arv_drug})
           WHERE o.order_type_id = 1 -- drug order
-            AND o.start_date < (DATE(#{start ? start_date : end_date }) + INTERVAL 1 DAY)
+            AND o.start_date < (DATE(#{start ? start_date : end_date}) + INTERVAL 1 DAY)
             AND o.voided = 0
           GROUP BY o.patient_id
           ON DUPLICATE KEY UPDATE start_date = VALUES(start_date), min_order_date = VALUES(min_order_date)
@@ -111,8 +110,8 @@ module MalawiHivProgramReports
           INNER JOIN drug_order PARTITION (p#{location}) ON drug_order.order_id = o.order_id AND drug_order.quantity > 0
             AND drug_order.drug_inventory_id IN (#{arv_drug})
           WHERE o.order_type_id = 1 -- drug order
-            AND o.start_date < (DATE(#{start ? start_date : end_date }) + INTERVAL 1 DAY)
-            AND o.start_date >= (DATE(#{start ? prev_date : start_date }) + INTERVAL 1 DAY)
+            AND o.start_date < (DATE(#{start ? start_date : end_date}) + INTERVAL 1 DAY)
+            AND o.start_date >= (DATE(#{start ? prev_date : start_date}) + INTERVAL 1 DAY)
             AND o.voided = 0
           GROUP BY o.patient_id
           ON DUPLICATE KEY UPDATE start_date = VALUES(start_date), min_order_date = VALUES(min_order_date)
@@ -323,8 +322,6 @@ module MalawiHivProgramReports
         SQL
       end
 
-      # rubocop:enable Metrics/MethodLength
-
       # ===================================
       #  Function Management Region
       # ===================================
@@ -342,15 +339,33 @@ module MalawiHivProgramReports
       def prepare_tables
         [false, true].each do |start|
           create_outcome_table(start:) unless check_if_table_exists("cdr_temp_patient_outcomes#{start ? '_start' : ''}")
-          drop_temp_patient_outcome_table(start:) unless count_table_columns("cdr_temp_patient_outcomes#{start ? '_start' : ''}")  == 5
-          create_tmp_max_drug_orders_table(start:) unless check_if_table_exists("cdr_temp_max_drug_orders#{start ? '_start' : ''}")
-          drop_temp_max_drug_orders_table(start:) unless count_table_columns("cdr_temp_max_drug_orders#{start ? '_start' : ''}") == 4
-          create_tmp_min_auto_expire_date(start:) unless check_if_table_exists("cdr_temp_min_auto_expire_date#{start ? '_start' : ''}")
-          drop_tmp_min_auto_expirte_date(start:) unless count_table_columns("cdr_temp_min_auto_expire_date#{start ? '_start' : ''}") == 6
-          create_cdr_temp_max_patient_state(start:) unless check_if_table_exists("cdr_temp_max_patient_state#{start ? '_start' : ''}")
-          create_temp_current_state(start:) unless check_if_table_exists("cdr_temp_current_state#{start ? '_start' : ''}")
-          drop_temp_current_state(start:) unless count_table_columns("cdr_temp_current_state#{start ? '_start' : ''}") == 7
-          create_temp_current_medication(start:) unless check_if_table_exists("cdr_temp_current_medication#{start ? '_start' : ''}")
+          unless count_table_columns("cdr_temp_patient_outcomes#{start ? '_start' : ''}") == 5
+            drop_temp_patient_outcome_table(start:)
+          end
+          unless check_if_table_exists("cdr_temp_max_drug_orders#{start ? '_start' : ''}")
+            create_tmp_max_drug_orders_table(start:)
+          end
+          unless count_table_columns("cdr_temp_max_drug_orders#{start ? '_start' : ''}") == 4
+            drop_temp_max_drug_orders_table(start:)
+          end
+          unless check_if_table_exists("cdr_temp_min_auto_expire_date#{start ? '_start' : ''}")
+            create_tmp_min_auto_expire_date(start:)
+          end
+          unless count_table_columns("cdr_temp_min_auto_expire_date#{start ? '_start' : ''}") == 6
+            drop_tmp_min_auto_expirte_date(start:)
+          end
+          unless check_if_table_exists("cdr_temp_max_patient_state#{start ? '_start' : ''}")
+            create_cdr_temp_max_patient_state(start:)
+          end
+          unless check_if_table_exists("cdr_temp_current_state#{start ? '_start' : ''}")
+            create_temp_current_state(start:)
+          end
+          unless count_table_columns("cdr_temp_current_state#{start ? '_start' : ''}") == 7
+            drop_temp_current_state(start:)
+          end
+          unless check_if_table_exists("cdr_temp_current_medication#{start ? '_start' : ''}")
+            create_temp_current_medication(start:)
+          end
         end
       end
 
@@ -390,7 +405,7 @@ module MalawiHivProgramReports
       end
 
       def drop_temp_patient_outcome_table(start: false)
-        ActiveRecord::Base.connection.execute <<~SQL 
+        ActiveRecord::Base.connection.execute <<~SQL
           DROP TABLE IF EXISTS cdr_temp_patient_outcomes#{start ? '_start' : ''}
         SQL
         create_outcome_table(start:)
@@ -451,13 +466,13 @@ module MalawiHivProgramReports
 
       def create_min_auto_expire_date_indexes(start: false)
         ActiveRecord::Base.connection.execute <<~SQL
-          CREATE INDEX idx_min_auto_expire_date#{start ? '_start' : ''} ON temp_min_auto_expire_date#{start ? '_start' : ''} (auto_expire_date)
+          CREATE INDEX idx_min_auto_expire_date#{start ? '_start' : ''} ON cdr_temp_min_auto_expire_date#{start ? '_start' : ''} (auto_expire_date)
         SQL
         ActiveRecord::Base.connection.execute <<~SQL
-          CREATE INDEX idx_min_pepfar#{start ? '_start' : ''} ON temp_min_auto_expire_date#{start ? '_start' : ''} (pepfar_defaulter_date)
+          CREATE INDEX idx_min_pepfar#{start ? '_start' : ''} ON cdr_temp_min_auto_expire_date#{start ? '_start' : ''} (pepfar_defaulter_date)
         SQL
         ActiveRecord::Base.connection.execute <<~SQL
-          CREATE INDEX idx_min_moh#{start ? '_start' : ''} ON temp_min_auto_expire_date#{start ? '_start' : ''} (moh_defaulter_date)
+          CREATE INDEX idx_min_moh#{start ? '_start' : ''} ON cdr_temp_min_auto_expire_date#{start ? '_start' : ''} (moh_defaulter_date)
         SQL
       end
 
@@ -560,7 +575,7 @@ module MalawiHivProgramReports
 
       def update_steps(start: false, portion: false)
         ActiveRecord::Base.connection.execute <<~SQL
-          UPDATE cdr_temp_patient_outcomes#{start ? '_start' : ''} SET step = 0 WHERE step >= #{portion ? 4 : 1 } AND site_id = #{location}
+          UPDATE cdr_temp_patient_outcomes#{start ? '_start' : ''} SET step = 0 WHERE step >= #{portion ? 4 : 1} AND site_id = #{location}
         SQL
       end
 
@@ -588,6 +603,5 @@ module MalawiHivProgramReports
       end
     end
     # rubocop:enable Metrics/ClassLength
-    # rubocop:enable Metrics/MethodLength
   end
 end

data/lib/malawi_hiv_program_reports/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MalawiHivProgramReports
-  VERSION = '1.1.0'
+  VERSION = '1.1.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: malawi_hiv_program_reports
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Roy Chanunkha
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-12 00:00:00.000000000 Z
+date: 2024-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails