renalware-core 2.0.0.pre.rc8 → 2.0.0.pre.rc9

data/db/functions/import_practice_memberships_csv_v01.sql

@@ -0,0 +1,48 @@
+CREATE OR REPLACE FUNCTION renalware.import_practice_memberships_csv(file text) RETURNS void
+  AS $$
+  BEGIN
+
+  DROP TABLE IF EXISTS memberships_via_copy;
+  CREATE TEMP TABLE copied_memberships (
+    gp_code text NOT NULL,
+    practice_code text NOT NULL,
+    unused3 text,
+    unused4 text,
+    unused5 text,
+    unused7 text
+  );
+
+  -- Import the CSV file into copied_memberships - note there is no CSV header in this file
+  EXECUTE format ('COPY copied_memberships FROM %L DELIMITER %L CSV ', file, ',');
+
+  DROP TABLE IF EXISTS tmp_memberships;
+  CREATE TEMP TABLE tmp_memberships AS
+    SELECT
+      gp_code,
+      practice_code,
+      patient_primary_care_physicians.id primary_care_physician_id,
+      patient_practices.id as practice_id
+      from copied_memberships
+      INNER JOIN patient_practices on patient_practices.code = practice_code
+      INNER JOIN patient_primary_care_physicians on patient_primary_care_physicians.code = gp_code;
+
+  -- Insert any new memberships, ignoring any conflicts where the
+  -- practice_id + primary_care_physician_id already exists
+  INSERT INTO renalware.patient_practice_memberships
+    (practice_id, primary_care_physician_id, created_at, updated_at)
+  SELECT
+    practice_id,
+    primary_care_physician_id,
+    CURRENT_TIMESTAMP,
+    CURRENT_TIMESTAMP
+  FROM tmp_memberships
+  ON CONFLICT (practice_id, primary_care_physician_id) DO NOTHING;
+
+  -- Mark as deleted any memberships not in the latest uploaded data set - ie those gps have retired or moved on
+  UPDATE patient_practice_memberships mem
+    SET deleted_at = CURRENT_TIMESTAMP
+    WHERE NOT EXISTS (select 1 FROM tmp_memberships tmem
+    WHERE tmem.practice_id = mem.practice_id AND tmem.primary_care_physician_id = mem.primary_care_physician_id);
+
+END;
+$$ LANGUAGE plpgsql;

data/db/functions/import_practices_csv_v01.sql

@@ -0,0 +1,109 @@
+CREATE OR REPLACE FUNCTION renalware.import_practices_csv(file text) RETURNS void AS $$
+  BEGIN
+  /*
+  Imports a practices.csv file created by parsing out an HSCOrgRefData_Full_xxxxx.xml file.
+  */
+  DROP TABLE IF EXISTS tmp_practices;
+
+  CREATE TEMP TABLE tmp_practices (
+    code text NOT NULL,
+    name text NOT NULL,
+    tel text,
+    street_1 text,
+    street_2 text,
+    street_3 text,
+    town text,
+    county text,
+    postcode text NOT NULL,
+    region text,
+    country_id integer,
+    active text NOT NULL,
+    CONSTRAINT tmp_practices_pkey PRIMARY KEY (code)
+  );
+
+  /* Import the CSV file into tmp_practices, ignoring the first row which is a header */
+  EXECUTE format ('COPY tmp_practices FROM %L DELIMITER %L CSV HEADER', file, ',');
+
+  /* Upsert practices */
+  WITH data(
+      code,
+      name,
+      telephone,
+      street_1,
+      street_2,
+      street_3,
+      town,
+      county,
+      postcode,
+      region,
+      country_id,
+      active)
+    AS (select * from tmp_practices)
+    , practice_changes AS (
+        INSERT INTO patient_practices (code, name, telephone, created_at, updated_at)
+        SELECT code, name, telephone, clock_timestamp(), clock_timestamp()
+        FROM data
+        ON CONFLICT (code) DO UPDATE
+          SET
+            name = excluded.name,
+            telephone = excluded.telephone,
+            updated_at = excluded.updated_at
+          RETURNING code, id
+      )
+
+  /* Upsert practice addresses */
+  INSERT INTO addresses (
+    addressable_type,
+    addressable_id,
+    street_1,
+    street_2,
+    street_3,
+    town,
+    county,
+    postcode,
+    region,
+    country_id,
+    created_at,
+    updated_at)
+  SELECT
+    'Renalware::Patients::Practice',
+    practice_changes.id,
+    street_1,
+    street_2,
+    street_3,
+    town,
+    county,
+    postcode,
+    region,
+    country_id,
+    CURRENT_TIMESTAMP,
+    CURRENT_TIMESTAMP
+    FROM data join practice_changes using(code)
+  ON CONFLICT (addressable_type, addressable_id) DO UPDATE
+    SET
+    street_1 = excluded.street_1,
+    street_2 = excluded.street_2,
+    street_3 = excluded.street_3,
+    town = excluded.town,
+    county = excluded.county,
+    postcode = excluded.postcode,
+    region = excluded.region,
+    country_id = excluded.country_id,
+    updated_at = clock_timestamp();
+
+  /* Update the deleted_at column of any practices which do not have an Active status_code */
+  UPDATE patient_practices AS p
+  SET deleted_at = CURRENT_TIMESTAMP
+  FROM tmp_practices AS tp
+  WHERE p.code = tp.code AND tp.active = 'false';
+
+  /* Set deleted_at tp NULL for active practices */
+  UPDATE patient_practices AS p
+  SET deleted_at = NULL
+  FROM tmp_practices AS tp
+  WHERE p.code = tp.code AND tp.active != 'false';
+
+  DROP TABLE tmp_practices;
+
+  END;
+  $$ LANGUAGE plpgsql;

data/db/functions/preprocess_hl7_message_v01.sql

@@ -0,0 +1,29 @@
+/* Create a function for the trigger to call */
+CREATE OR REPLACE FUNCTION preprocess_hl7_message() RETURNS trigger AS
+$body$
+BEGIN
+  /*
+  Mirth inserts a row into delayed job when a new HL7 message needs to be processed by Renalware.
+  The SQL it uses looks like this:
+    insert into renalware.delayed_jobs (handler, run_at)
+    values(E'--- !ruby/struct:FeedJob\nraw_message: |\n  ' || REPLACE(${message.rawData},E'\r',E'\n  '), NOW());
+  This works unless there is a 10^12 value in the unit of measurement segment for an OBX (e.g.
+  for WBC or HGB). Then Mirth encodes the ^ as \S\ because ^ is a significant character in Mirth
+  (field separator). Unfortunately this creates the combination
+  10\S\12 and S\12 is converted to \n when the handler's payload is loaded in by the delayed_job worker.
+  To get around this we need to convert instances of \S\ with another escape sequence eg «
+  and manually map this back to a ^ in the job handler ruby code.
+
+  So here, if this delayed_job is destined to be picked up by a Feed job handler
+  make sure we convert the Mirth escape sequence \S\ to \\S\\
+  */
+  IF position('Feed' in NEW.handler) > 0 THEN
+    NEW.handler = replace(NEW.handler, E'\\S\\', E'\\\\S\\\\');
+  END IF;
+
+  RETURN NEW;
+END
+
+$body$
+LANGUAGE plpgsql;
+

data/db/functions/preprocess_hl7_message_v02.sql

@@ -0,0 +1,31 @@
+/* Create a function for the trigger to call */
+CREATE OR REPLACE FUNCTION preprocess_hl7_message() RETURNS trigger AS
+$body$
+BEGIN
+  /*
+  Mirth inserts a row into delayed job when a new HL7 message needs to be processed by Renalware.
+  The SQL it uses looks like this:
+    insert into renalware.delayed_jobs (handler, run_at)
+    values(E'--- !ruby/struct:FeedJob\nraw_message: |\n  ' || REPLACE(${message.rawData},E'\r',E'\n  '), NOW());
+  This works unless there is a 10^12 value in the unit of measurement segment for an OBX (e.g.
+  for WBC or HGB). Then Mirth encodes the ^ as \S\ because ^ is a significant character in Mirth
+  (field separator). Unfortunately this creates the combination
+  10\S\12 and S\12 is converted to \n when the handler's payload is loaded in by the delayed_job worker.
+  To get around this we need to convert instances of \S\ with another escape sequence eg «
+  and manually map this back to a ^ in the job handler ruby code.
+
+  So here, if this delayed_job is destined to be picked up by a Feed job handler
+  make sure we convert the Mirth escape sequence \S\ to \\S\\
+  */
+  IF position('Feed' in NEW.handler) > 0 THEN
+    NEW.handler = replace(NEW.handler, E'\\S\\', E'\\\\S\\\\');
+    NEW.created_at = now();
+    NEW.updated_at = now();
+  END IF;
+
+  RETURN NEW;
+END
+
+$body$
+LANGUAGE plpgsql;
+

data/db/functions/refresh_all_matierialized_views_v01.sql

@@ -0,0 +1,32 @@
+CREATE OR REPLACE FUNCTION
+  refresh_all_matierialized_views(_schema TEXT DEFAULT '*', _concurrently BOOLEAN DEFAULT false)
+  /*
+  Function to refresh all materialized views in all or a specified schema,
+  optionally concurrently
+  From https://github.com/frankhommers/RefreshAllMaterializedViews
+  */
+  RETURNS INT AS $$
+    DECLARE
+      r RECORD;
+    BEGIN
+      RAISE NOTICE 'Refreshing materialized view(s) in % %',
+        CASE WHEN _schema = '*' THEN 'all schemas'
+        ELSE 'schema "'|| _schema || '"'
+        END,
+        CASE WHEN _concurrently
+        THEN 'concurrently'
+        ELSE '' END;
+      IF pg_is_in_recovery() THEN
+        RETURN 0;
+      ELSE
+        FOR r IN SELECT schemaname,
+                        matviewname FROM pg_matviews WHERE schemaname = _schema OR _schema = '*'
+        LOOP
+          RAISE NOTICE 'Refreshing materialized view "%"."%"', r.schemaname, r.matviewname;
+          EXECUTE 'REFRESH MATERIALIZED VIEW ' || CASE WHEN _concurrently THEN 'CONCURRENTLY '
+          ELSE '' END || '"' || r.schemaname || '"."' || r.matviewname || '"';
+        END LOOP;
+      END IF;
+      RETURN 1;
+    END
+  $$ LANGUAGE plpgsql;

data/db/functions/refresh_current_observation_set_v01.sql

@@ -0,0 +1,39 @@
+CREATE OR REPLACE FUNCTION refresh_current_observation_set(a_patient_id integer)
+  -- Function to update the pathology_current_observation_set for a patient.
+  -- It stores the most recent results into the jsonb hash on that table.
+  -- To run for all patients, use
+  --   select refresh_current_observation_set(id) from patients;
+  --
+    RETURNS integer
+    LANGUAGE 'plpgsql'
+    as $$
+  BEGIN
+  with current_patient_obs as(
+      select
+        DISTINCT ON (p.id, obxd.id)
+  p.id as patient_id,
+        obxd.code,
+        json_build_object('result',(obx.result),'observed_at',obx.observed_at) as value
+        from patients p
+        inner join pathology_observation_requests obr on obr.patient_id = p.id
+        inner join pathology_observations obx on obx.request_id = obr.id
+        inner join pathology_observation_descriptions obxd on obx.description_id = obxd.id
+        where p.id = a_patient_id
+        order by p.id, obxd.id, obx.observed_at desc
+    ),
+    current_patient_obs_as_jsonb as (
+      select patient_id,
+        jsonb_object_agg(code, value) as values,
+        CURRENT_TIMESTAMP,
+        CURRENT_TIMESTAMP
+        from current_patient_obs
+        group by patient_id order by patient_id
+    )
+    insert into pathology_current_observation_sets (patient_id, values, created_at, updated_at)
+      select * from current_patient_obs_as_jsonb
+      ON conflict (patient_id)
+      DO UPDATE
+      SET values = excluded.values, updated_at = excluded.updated_at;
+  RETURN a_patient_id;
+END
+$$;

data/db/functions/update_current_observation_set_from_trigger_v01.sql

@@ -0,0 +1,88 @@
+CREATE OR REPLACE FUNCTION update_current_observation_set_from_trigger() RETURNS TRIGGER AS $body$
+-- TC 14/12/2017 v01
+-- This function is called by a trigger when a row is inserted or updated in
+-- pathology_observations. Its purpose is to keep current_observation_sets up to date
+-- with the latest observations for any patient.
+-- The current_observation_sets table maintains a jsonb hash into which we insert or replace
+-- the observation, keyed by OBX code.
+-- e.g.  .. {"HGB": { "result": 123.1, "observed_at": '2017-12-12-01:01:01'}, ..
+DECLARE
+  a_patient_id bigint;
+  a_code text;
+  current_observed_at timestamp;
+  current_result text;
+  new_observed_at timestamp;
+BEGIN
+  RAISE NOTICE 'TRIGGER called on %',TG_TABLE_NAME ;
+
+  /*
+  If inserting or updating, we _could_ assume the last observation to be inserted is
+  the most 'recent' one (with the latest observed_at date).
+  However the order of incoming messages is not guaranteed, so we have two options:
+  1. Refresh the entire current_observation_set for the patient
+  2. Check the current observed_at date in the jsonb and only update if we have a more
+     recent one
+  We have gone for 2.
+  */
+
+  IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
+
+    -- Note we could re-generate the entire current pathology for the patient using
+    --  select refresh_current_observation_set(a_patient_id);
+    -- which is safer but uses more resources, so avoiding this for now.
+
+    -- Find and store patient_id into local variable
+    select request.patient_id into a_patient_id
+      from pathology_observation_requests request
+      where request.id = NEW.request_id;
+
+    -- Find and store the obx code into local variable
+    select description.code into a_code
+      from pathology_observation_descriptions description
+      where description.id = NEW.description_id;
+
+    -- Important! Create the observation_set if it doesn exist yet
+    -- ignore the error id the row already exists
+    insert into pathology_current_observation_sets (patient_id)
+    values (a_patient_id)
+    ON CONFLICT DO NOTHING;
+
+    -- We are going to compare the current and new observed_at dates
+    -- so need to cast them  to a timestamp
+    select cast(New.observed_at as timestamp) into new_observed_at;
+
+    -- Get the most recent date and value for this observation
+    -- and store to variables.
+    select
+    cast(values -> a_code ->> 'observed_at' as timestamp),
+    values -> a_code ->> 'result'
+    into current_observed_at, current_result from
+    pathology_current_observation_sets
+    where patient_id = a_patient_id;
+
+    -- Output some info to helps us debug. This can be removed later.
+    RAISE NOTICE '  Request id % Patient id % Code %', NEW.request_id, a_patient_id, a_code;
+    RAISE NOTICE '  Last %: % at %', a_code, current_result, current_observed_at;
+    RAISE NOTICE '  New  %: % at %', a_code, NEW.result, new_observed_at;
+
+    IF current_observed_at IS NULL OR new_observed_at >= current_observed_at THEN
+      -- The new pathology_observation row contain a more recent result that the old one.
+      -- (note there may not be an old one if the patient has neve had this obs before).
+
+      RAISE NOTICE '  Updating pathology_current_observation_sets..';
+
+      -- Update the values jsonb column with the new hash for this code, e.g.
+      -- .. {"HGB": { "result": 123.1, "observed_at": '2017-12-12-01:01:01'}, ..
+            -- Note the `set values` below actually reads in the jsonb, updates it,
+            -- and wites the whole thing back.
+      update pathology_current_observation_sets
+        set values = jsonb_set(
+          values,
+          ('{'||a_code||'}')::text[], -- defined in the fn path::text[]
+          jsonb_build_object('result', NEW.result, 'observed_at', new_observed_at),
+          true)
+        where patient_id = a_patient_id;
+    END IF;
+  END IF;
+  RETURN NULL ;
+END $body$ LANGUAGE plpgsql VOLATILE COST 100;

data/db/functions/update_current_observation_set_from_trigger_v02.sql

@@ -0,0 +1,90 @@
+CREATE OR REPLACE FUNCTION update_current_observation_set_from_trigger() RETURNS TRIGGER AS $body$
+-- TC 14/12/2017 v02
+-- This function is called by a trigger when a row is inserted or updated in
+-- pathology_observations. Its purpose is to keep current_observation_sets up to date
+-- with the latest observations for any patient.
+-- The current_observation_sets table maintains a jsonb hash into which we insert or replace
+-- the observation, keyed by OBX code.
+-- e.g.  .. {"HGB": { "result": 123.1, "observed_at": '2017-12-12-01:01:01'}, ..
+DECLARE
+  a_patient_id bigint;
+  a_code text;
+  current_observed_at timestamp;
+  current_result text;
+  new_observed_at timestamp;
+BEGIN
+  RAISE NOTICE 'TRIGGER called on %',TG_TABLE_NAME ;
+
+  /*
+  If inserting or updating, we _could_ assume the last observation to be inserted is
+  the most 'recent' one (with the latest observed_at date).
+  However the order of incoming messages is not guaranteed, so we have two options:
+  1. Refresh the entire current_observation_set for the patient
+  2. Check the current observed_at date in the jsonb and only update if we have a more
+     recent one
+  We have gone for 2.
+  */
+
+  IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
+
+    -- Note we could re-generate the entire current pathology for the patient using
+    --  select refresh_current_observation_set(a_patient_id);
+    -- which is safer but uses more resources, so avoiding this for now.
+
+    -- Find and store patient_id into local variable
+    select request.patient_id into a_patient_id
+      from pathology_observation_requests request
+      where request.id = NEW.request_id;
+
+    -- Find and store the obx code into local variable
+    select description.code into a_code
+      from pathology_observation_descriptions description
+      where description.id = NEW.description_id;
+
+    -- Important! Create the observation_set if it doesn't exist yet
+    -- ignore the error if the row already exists
+    insert into pathology_current_observation_sets (patient_id)
+    values (a_patient_id)
+    ON CONFLICT DO NOTHING;
+
+    -- We are going to compare the current and new observed_at dates
+    -- so need to cast them  to a timestamp
+    select cast(New.observed_at as timestamp) into new_observed_at;
+
+    -- Get the most recent date and value for this observation
+    -- and store to variables.
+    select
+    cast(values -> a_code ->> 'observed_at' as timestamp),
+    values -> a_code ->> 'result'
+    into current_observed_at, current_result from
+    pathology_current_observation_sets
+    where patient_id = a_patient_id;
+
+    -- Output some info to helps us debug. This can be removed later.
+    RAISE NOTICE '  Request id % Patient id % Code %', NEW.request_id, a_patient_id, a_code;
+    RAISE NOTICE '  Last %: % at %', a_code, current_result, current_observed_at;
+    RAISE NOTICE '  New  %: % at %', a_code, NEW.result, new_observed_at;
+
+    IF current_observed_at IS NULL OR new_observed_at >= current_observed_at THEN
+      -- The new pathology_observation row contain a more recent result that the old one.
+      -- (note there may not be an old one if the patient has neve had this obs before).
+
+      RAISE NOTICE '  Updating pathology_current_observation_sets..';
+
+      -- Update the values jsonb column with the new hash for this code, e.g.
+      -- .. {"HGB": { "result": 123.1, "observed_at": '2017-12-12-01:01:01'}, ..
+            -- Note the `set values` below actually reads in the jsonb, updates it,
+            -- and wites the whole thing back.
+      update pathology_current_observation_sets
+        set updated_at = CURRENT_TIMESTAMP,
+          values = jsonb_set(
+          values,
+          ('{'||a_code||'}')::text[], -- defined in the fn path::text[]
+          jsonb_build_object('result', NEW.result, 'observed_at', new_observed_at),
+          true)
+        where patient_id = a_patient_id;
+    END IF;
+  END IF;
+  RETURN NULL ;
+END $body$ LANGUAGE plpgsql VOLATILE COST 100;
+-- End function