his_emr_api_lab 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 778184a052cac6932ec700f03b99615024440af3944c371d0cfa997f357c525c
-  data.tar.gz: 045fb73daab53abc457c08152c5148251a6dccb239f0c76e09f3172007e44b6e
+  metadata.gz: f53ca7acc69718ba31cf2ae278b6fdbdc4996d3ad6a28db0a08d3945d1463998
+  data.tar.gz: 77b2cfdc430706b9864e9ec71b2f4b64c2ec11a5695b45b81984c86a8fefefd4
 SHA512:
-  metadata.gz: 95f56b7a5d1b36565074904d49e5ff69b34fb57bed690a7bec5d269749c6813e094b4e02b397517523edd90e0714710376f489338788838fad775f3c724659db
-  data.tar.gz: 380096a9132eb857a4a321260593640eec0d6a5b08f661e97a1ba2396619eb5984c1aca488e364930dff737ff397b07865597b9ed321c7a93101c151a8d2566f
+  metadata.gz: 19f53f3c471dd4552f64bf1200a6bbd8051e7bc7bc7e0670b2079bea4af61621575a9c0e7f5cdc06c43ac7b80c2534e6ca704fd06e43b0deed3475ea66690017
+  data.tar.gz: cbe4e02964f635b60e7e7b0c6b53ac3b406ab9bff55da582371a3de38a8d65901bcca1771a7b3cf0d8be83e31ac8bf0a15e9c752f7131bb631f1011df95fa713

data/app/services/lab/lims/api.rb

@@ -29,16 +29,11 @@ module Lab
       # given block until the queue is empty or connection is terminated
       # by calling method +choke+.
       def consume_orders(from: 0, limit: 30)
-        last_seq = { value: 0 }
-
         bum.binge_changes(since: from, limit: limit, include_docs: true) do |change|
           next unless change['doc']['type']&.casecmp?('Order')
 
           yield OrderDTO.new(change['doc']), self
-          last_seq[:value] = self.last_seq
         end
-
-        last_seq[:value]
       end
 
       def create_order(order)

data/app/services/lab/lims/migrator.rb

@@ -29,6 +29,7 @@ require 'lab/lab_test'
 require 'lab/lims_order_mapping'
 require 'lab/lims_failed_import'
 
+require_relative './worker'
 require_relative '../orders_service'
 require_relative '../results_service'
 require_relative '../tests_service'
@@ -47,14 +48,16 @@ module Lab
 
         attr_reader :rejections
 
-        def consume_orders(from: nil, limit: 50_000)
+        def consume_orders(from: nil, **_kwargs)
+          limit = 50_000
+
           Parallel.each(read_orders(from, limit),
                         in_processes: MAX_THREADS,
                         finish: order_pmap_post_processor(from)) do |row|
             next unless row['doc']['type']&.casecmp?('Order')
 
-            User.current = Migrator.lab_user
-            yield OrderDTO.new(row['doc']), OpenStruct.new(last_seq: from)
+            User.current = Utils.lab_user
+            yield OrderDTO.new(row['doc']), OpenStruct.new(last_seq: (from || 0) + limit, current_seq: from)
           end
         end
 
@@ -70,7 +73,7 @@ module Lab
         private
 
         def last_seq_path
-          Rails.root.join('log/lims/migration-last-id.dat')
+          LIMS_LOG_PATH.join('migration-last-id.dat')
         end
 
         def order_pmap_post_processor(last_seq)
@@ -129,18 +132,6 @@ module Lab
         def update_last_seq(_last_seq); end
       end
 
-      def self.lab_user
-        user = User.find_by_username('lab_daemon')
-        return user if user
-
-        god_user = User.first
-
-        person = Person.create!(creator: god_user.user_id)
-        PersonName.create!(person: person, given_name: 'Lab', family_name: 'Daemon', creator: god_user.user_id)
-
-        User.create!(username: 'lab_daemon', person: person, creator: god_user.user_id)
-      end
-
       def self.save_csv(filename, rows:, headers: nil)
         CSV.open(filename, File::WRONLY | File::CREAT) do |csv|
           csv << headers if headers
@@ -148,7 +139,7 @@ module Lab
         end
       end
 
-      MIGRATION_REJECTIONS_CSV_PATH = Rails.root.join('log/lims/migration-rejections.csv')
+      MIGRATION_REJECTIONS_CSV_PATH = LIMS_LOG_PATH.join('migration-rejections.csv')
 
       def self.export_rejections(rejections)
         headers = ['Accession number', 'NHID', 'First name', 'Last name', 'Reason']
@@ -165,7 +156,7 @@ module Lab
         save_csv(MIGRATION_REJECTIONS_CSV_PATH, headers: headers, rows: rows)
       end
 
-      MIGRATION_FAILURES_CSV_PATH = Rails.root.join('log/lims/migration-failures.csv')
+      MIGRATION_FAILURES_CSV_PATH = LIMS_LOG_PATH.join('migration-failures.csv')
 
       def self.export_failures
         headers = ['Accession number', 'NHID', 'Reason', 'Difference']
@@ -181,7 +172,7 @@ module Lab
         save_csv(MIGRATION_FAILURES_CSV_PATH, headers: headers, rows: rows)
       end
 
-      MIGRATION_LOG_PATH = Rails.root.join('log/lims/migration.log')
+      MIGRATION_LOG_PATH = LIMS_LOG_PATH.join('migration.log')
 
       def self.start_migration
         log_dir = Rails.root.join('log/lims')

data/app/services/lab/lims/order_serializer.rb

@@ -97,7 +97,9 @@ module Lab
         end
 
         def format_test_results(order)
-          order.tests.each_with_object({}) do |test, results|
+          order.tests&.each_with_object({}) do |test, results|
+            next unless test.result
+
             results[test.name] = {
               results: test.result.each_with_object({}) do |measure, measures|
                 measures[measure.indicator.name] = { result_value: "#{measure.value_modifier}#{measure.value}" }

data/app/services/lab/lims/utils.rb

@@ -39,6 +39,18 @@ module Lab
         end
       end
 
+      def self.lab_user
+        user = User.find_by_username('lab_daemon')
+        return user if user
+
+        god_user = User.first
+
+        person = Person.create!(creator: god_user.user_id)
+        PersonName.create!(person: person, given_name: 'Lab', family_name: 'Daemon', creator: god_user.user_id)
+
+        User.create!(username: 'lab_daemon', person: person, creator: god_user.user_id)
+      end
+
       def self.parse_date(str_date, fallback_date = nil)
         if str_date.blank? && fallback_date.blank?
           raise "Can't parse blank date"

data/app/services/lab/lims/worker.rb

@@ -2,12 +2,16 @@
 
 require 'cgi/util'
 
+require_relative './api'
 require_relative './exceptions'
 require_relative './order_serializer'
 require_relative './utils'
 
 module Lab
   module Lims
+    LIMS_LOG_PATH = Rails.root.join('log/lims')
+    Dir.mkdir(LIMS_LOG_PATH) unless File.exist?(LIMS_LOG_PATH)
+
     ##
     # Pull/Push orders from/to the LIMS queue (Oops meant CouchDB).
     class Worker
@@ -15,6 +19,19 @@ module Lab
 
       attr_reader :lims_api
 
+      def self.start
+        File.open(LIMS_LOG_PATH.join('worker.lock'), File::WRONLY | File::CREAT, 0o644) do |fout|
+          fout.flock(File::LOCK_EX)
+
+          User.current = Utils.lab_user
+
+          fout.write("Worker ##{Process.pid} started at #{Time.now}")
+          worker = new(Api.new)
+          worker.pull_orders
+          worker.push_orders
+        end
+      end
+
       def initialize(lims_api)
         @lims_api = lims_api
       end
@@ -52,8 +69,8 @@ module Lab
             lims_api.update_order(mapping.lims_id, order_dto)
             mapping.update(pushed_at: Time.now)
           else
-            order_dto = lims_api.create_order(order_dto)
-            LimsOrderMapping.create!(order: order, lims_id: order_dto['_id'], pushed_at: Time.now)
+            update = lims_api.create_order(order_dto)
+            LimsOrderMapping.create!(order: order, lims_id: update['id'], revision: update['rev'], pushed_at: Time.now)
           end
         end
 
@@ -64,7 +81,8 @@ module Lab
       # Pulls orders from the LIMS queue and writes them to the local database
       def pull_orders
         logger.info("Retrieving LIMS orders starting from #{last_seq}")
-        lims_api.consume_orders(from: last_seq) do |order_dto, context|
+
+        lims_api.consume_orders(from: last_seq, limit: 100) do |order_dto, context|
           logger.debug("Retrieved order ##{order_dto[:tracking_number]}: #{order_dto}")
 
           patient = find_patient_by_nhid(order_dto[:patient][:id])
@@ -80,6 +98,8 @@ module Lab
             save_failed_import(order_dto, 'Demographics not matching', diff)
           end
 
+          update_last_seq(context.current_seq)
+
           [:accepted, "Patient NPID, '#{order_dto[:patient][:id]}', matched"]
         rescue DuplicateNHID
           logger.warn("Failed to import order due to duplicate patient NHID: #{order_dto[:patient][:id]}")
@@ -90,8 +110,6 @@ module Lab
         rescue LimsException => e
           logger.warn("Failed to import order due to #{e.class} - #{e.message}")
           save_failed_import(order_dto, e.message)
-        ensure
-          update_last_seq(context.last_seq)
         end
       end
 
@@ -303,7 +321,7 @@ module Lab
       end
 
       def last_seq_path
-        Rails.root.join('log/lims/last-seq.dat')
+        LIMS_LOG_PATH.join('last_seq.dat')
       end
     end
   end

data/lib/couch_bum/couch_bum.rb

@@ -24,18 +24,21 @@ class CouchBum
   # within the passed block.
   def binge_changes(since: 0, limit: nil, include_docs: nil, &block)
     catch(:choke) do
-      extra_params = stringify_params(limit: limit, include_docs: include_docs)
+      logger.debug("Binging #{limit} changes from '#{since}'")
+      params = stringify_params(limit: limit, include_docs: include_docs)
+      params = "since=#{since}&#{params}" unless since.blank?
 
-      changes = couch_rest(:get, "_changes?since=#{since}&#{extra_params}")
+      changes = couch_rest(:get, "_changes?#{params}")
       context = BingeContext.new(changes)
-      changes['results'].each { |change| context.instance_exec(change, &block) }
+      changes['results'].each do |change|
+        context.current_seq = change['seq']
+        context.instance_exec(change, &block)
+      end
     end
   end
 
   def couch_rest(method, route, *args, **kwargs)
     url = expand_route(route)
-
-    logger.debug("CouchBum: Executing #{method} #{url}")
     CouchRest.send(method, url, *args, **kwargs)
   rescue CouchRest::Exception => e
     logger.error("Failed to communicate with CouchDB: Status: #{e.http_code} - #{e.http_body}")
@@ -46,6 +49,8 @@ class CouchBum
 
   # Context under which the callback passed to binge_changes is executed.
   class BingeContext
+    attr_accessor :current_seq
+
     def initialize(changes)
       @changes = changes
     end

data/lib/lab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lab
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: his_emr_api_lab
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Elizabeth Glaser Pediatric Foundation Malawi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-18 00:00:00.000000000 Z
+date: 2021-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: couchrest