malawi_hiv_program_reports 1.1.15 → 1.1.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7648d6a581512a4f62341d93358beb5dd3895ed261d97aa849e0f143bb16645d
-  data.tar.gz: 87f0a1eb2a33f5839a97bc9af592b9a4f18bbf15236406b56a163ae0e2641de1
+  metadata.gz: 394a402d0b5e0c9b210bbf5feff0959854bb2b8d1322ecd97b07107201dcd694
+  data.tar.gz: 51615fcd686c06088e928b6b10e1bba26a754825e9c3d891e0e7c43c00647098
 SHA512:
-  metadata.gz: 803bdb400dd06a631cc42c0712f507fc7750e7922c6a9709e13820037bc54f9dbae24360b6d93465f400ca59cb2a609c4dd015f05def601c15742cb6b4e22a8b
-  data.tar.gz: 365656a38cf502c597cb4536f3aff8adb95241b176f2f3be20ebc6e42a4e8e7f861324a9ecdb35839d7c96157b433f85c31bc4868c6f7b93f59238e0528dd4ff
+  metadata.gz: aa30a38678f9f6dbeded4d8b3fe6ae920a46c6bea63937906f330cdf50b2cbc1ca6ae95edcf004caa2b4ca5501c6dbf627210f2f496dd49e1ba3964815e3a28f
+  data.tar.gz: cccee12cb7e4e5d9e5e685230b63309f93ad6db7199de3ae244d461a95e0f825f71a0d82f9b01a535636ad2c45fffc617dfcc68460d1f9508853c059ee71be9e

data/app/services/malawi_hiv_program_reports/moh/cumulative_cohort.rb

@@ -7,10 +7,11 @@ module MalawiHivProgramReports
     class CumulativeCohort
       include MalawiHivProgramReports::Utils::CommonSqlQueryUtils
 
-      attr_reader :start_date, :end_date, :locations, :rebuild, :outcome, :definition
+      attr_reader :start_date, :end_date, :locations, :rebuild, :outcome, :definition, :iteration, :redis, :start_time
 
       LOCK_FILE = 'art_service/reports/cumulative_cohort.lock'
 
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       def initialize(start_date:, end_date:, **kwargs)
         @start_date = ActiveRecord::Base.connection.quote(start_date)
         @end_date = ActiveRecord::Base.connection.quote(end_date)
@@ -18,13 +19,17 @@ module MalawiHivProgramReports
         locations = kwargs[:locations]
         @locations = locations.present? ? locations.split(',') : []
         @definition = kwargs[:definition] || 'pepfar'
-
+        @iteration = Concurrent::AtomicFixnum.new(0)
+        @start_time = Time.now
+        @redis = Redis.new
+        @redis.set('cumulative_progress', 0)
+        @redis.set('cumulative_status', 'running')
+        @redis.set('cumulative_time_taken', 0)
         definition = @definition.downcase
         raise ArgumentError, "Invalid outcomes definition: #{definition}" unless %w[moh pepfar].include?(definition)
       end
 
       def find_report
-        start_time = Time.now
         handle_tables
         clear_cohort_status unless rebuild
         process_thread(locations:)
@@ -33,8 +38,11 @@ module MalawiHivProgramReports
         handle_failed
         time_taken_for_failed = ((Time.now - end_time) / 60).round(2)
         Rails.logger.info("Cumulative Cohort report took #{time_in_minutes} minutes to generate for these locations: #{locations}")
+        redis.set('cumulative_time_taken', time_in_minutes)
+        redis.set('cumulative_status', 'completed')
         { cohort_time: time_in_minutes, processing_failed: time_taken_for_failed }
       end
+      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
       private
 
@@ -46,13 +54,12 @@ module MalawiHivProgramReports
       # 5. We want to get information on clients birthdate etc after joining all the above
       # 6. Finally we want to get the outcomes of the clients in 5 above
 
-      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       def process_thread(locations: [])
-        # use locations to thread but process 10 locations at a time
         queue = Queue.new
         locations.each { |loc| queue << loc }
 
-        threads = Array.new(40) do
+        threads = Array.new(65) do
           Thread.new do
             until queue.empty?
               loc = begin
@@ -64,6 +71,8 @@ module MalawiHivProgramReports
 
               ActiveRecord::Base.connection_pool.with_connection do
                 process_data loc
+                current_iteration = iteration.increment
+                update_progress(current_iteration, locations.size)
               rescue StandardError => e
                 Rails.logger.info("Error processing location #{loc}: #{e.message}")
                 Rails.logger.info(e.backtrace.join("\n"))
@@ -75,7 +84,18 @@ module MalawiHivProgramReports
 
         threads.each(&:join)
       end
-      # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+      # update progress in redis and channel
+      def update_progress(current_iteration, total)
+        percentage = (current_iteration.to_f / total * 100).round(2)
+        redis.set('cumulative_progress', percentage)
+        ActionCable.server.broadcast('cumulative_updates_channel', {
+                                       progress: percentage,
+                                       status: 'running',
+                                       time_taken: ((Time.now - start_time) / 60).round(2)
+                                     })
+      end
 
       def reprocess_failed
         failed_locs = failed_sites&.map { |loc| loc[:site_id] }
@@ -84,6 +104,7 @@ module MalawiHivProgramReports
         process_thread(locations: failed_locs)
       end
 
+      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       def handle_failed
         failed_locs = failed_sites&.map { |loc| loc['site_id'].to_i }
         return if failed_locs.empty?
@@ -102,9 +123,13 @@ module MalawiHivProgramReports
             Rails.logger.info("Error processing location #{loc}: #{e.message}")
             Rails.logger.info(e.backtrace.join("\n"))
             save_incomplete_site(location: loc, time_taken: 0)
+          ensure
+            current_iteration = iteration.increment
+            update_progress(current_iteration, locations.size)
           end
         end
       end
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
       def failed_sites
         ActiveRecord::Base.connection.select_all <<~SQL

data/app/services/malawi_hiv_program_reports/moh/cumulative_outcome.rb

@@ -317,7 +317,7 @@ module MalawiHivProgramReports
           INSERT INTO cdr_temp_patient_outcomes#{start ? '_start' : ''} PARTITION (p#{location})
           SELECT patient_id, #{patient_outcome_function('patient_id', 'site_id', start)}, NULL, #{location}, 6
           FROM cdr_temp_cohort_members PARTITION (p#{location})
-          WHERE date_enrolled <= #{end_date}
+          WHERE date_enrolled <=  DATE(#{start ? start_date : end_date}) #{start ? '- INTERVAL 1 DAY' : ''}
           AND (patient_id, site_id) NOT IN (SELECT patient_id, site_id FROM cdr_temp_patient_outcomes#{start ? '_start' : ''} PARTITION (p#{location}) WHERE step IN (1, 2, 3, 4, 5))
           ON DUPLICATE KEY UPDATE cum_outcome = VALUES(cum_outcome), outcome_date = VALUES(outcome_date), step = VALUES(step)
         SQL
@@ -586,7 +586,7 @@ module MalawiHivProgramReports
 
       def clear_tables(start: false)
         # we truncate if @locations is empty otherwise we only clean the specifieds
-        if location.empty?
+        if location.blank?
           ActiveRecord::Base.connection.execute("TRUNCATE cdr_temp_patient_outcomes#{start ? '_start' : ''}")
           ActiveRecord::Base.connection.execute("TRUNCATE cdr_temp_max_drug_orders#{start ? '_start' : ''}")
           ActiveRecord::Base.connection.execute("TRUNCATE cdr_temp_min_auto_expire_date#{start ? '_start' : ''}")

data/lib/malawi_hiv_program_reports/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MalawiHivProgramReports
-  VERSION = '1.1.15'
+  VERSION = '1.1.16'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: malawi_hiv_program_reports
 version: !ruby/object:Gem::Version
-  version: 1.1.15
+  version: 1.1.16
 platform: ruby
 authors:
 - Roy Chanunkha
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-23 00:00:00.000000000 Z
+date: 2024-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails