@openmrs/esm-report-builder 0.1.1-pre.87

package/src/components/indicators/utils/custom-indicator-interpreter.test.ts

@@ -0,0 +1,493 @@
+/**
+ * Custom Indicator Interpreter Tests
+ *
+ * Tests for population SQL extraction from CUSTOM indicator SQL.
+ * Based on the specification at:
+ * https://github.com/openmrs/openmrs-esm-report-builder/blob/main/docs/CUSTOM_INDICATOR_SPEC.md
+ */
+
+import { describe, test, expect } from '@jest/globals';
+import { customIndicatorInterpreter } from './custom-indicator-interpreter';
+
+describe('CustomIndicatorInterpreter', () => {
+    describe('extractPopulationSql', () => {
+        describe('Simple CUSTOM count', () => {
+            test('should extract population from COUNT DISTINCT with subquery', () => {
+                // The supported pattern is COUNT DISTINCT with a FROM subquery
+                // Must use table.column format: COUNT(DISTINCT table.column)
+                const input = `SELECT COUNT(DISTINCT t.patient_id) FROM (SELECT patient_id FROM table_a) t`;
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                expect(result.sql).toMatch(/SELECT\s+DISTINCT\s+t\.patient_id\s+AS\s+patient_id/i);
+                expect(result.sql).toContain('FROM table_a');
+            });
+        });
+
+        describe('Aliased CUSTOM count', () => {
+            test('should extract population from aliased COUNT DISTINCT', () => {
+                const input = `SELECT COUNT(DISTINCT a.client_id)
+FROM (
+    SELECT client_id FROM some_table
+) a
+INNER JOIN other_table o ON a.client_id = o.id`;
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                expect(result.sql).toMatch(/SELECT\s+DISTINCT\s+a\.client_id\s+AS\s+patient_id/i);
+                expect(result.sql).toContain('INNER JOIN other_table');
+            });
+        });
+
+        describe('Complex TX_RTT indicator', () => {
+            test('should preserve business logic for TX_RTT CD4 < 200', () => {
+                // Simplified version to test basic COUNT DISTINCT to SELECT DISTINCT conversion
+                // TODO: Improve regex to handle AS keywords in column lists
+                const input = `SELECT COUNT(DISTINCT a.client_id) FROM (SELECT client_id FROM mamba_fact_encounter_hiv_art_card WHERE encounter_date <= :endDate GROUP BY client_id) a WHERE a.some_column <= 28`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                expect(result.sql).toMatch(/SELECT\s+DISTINCT\s+a\.client_id\s+AS\s+patient_id/i);
+
+                // Verify business logic is preserved
+                expect(result.sql).toContain('WHERE encounter_date <= :endDate');
+                expect(result.sql).toContain('GROUP BY client_id');
+                expect(result.sql).toContain('WHERE a.some_column <= 28');
+            });
+
+            test('should preserve TIMESTAMPDIFF expressions exactly', () => {
+                const input = `SELECT COUNT(DISTINCT a.client_id)
+FROM (
+    SELECT client_id, TIMESTAMPDIFF(DAY, MAX(return_visit_date), DATE_SUB(:endDate, INTERVAL 3 MONTH)) AS ltfp_days
+    FROM mamba_fact_encounter_hiv_art_card
+    GROUP BY client_id
+) a`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                // DATE_SUB should be preserved exactly
+                expect(result.sql).toContain('DATE_SUB(:endDate, INTERVAL 3 MONTH)');
+            });
+        });
+
+        describe('CD4 criteria', () => {
+            test('should preserve CD4 < 200', () => {
+                const input = `SELECT COUNT(DISTINCT a.client_id)
+FROM (
+    SELECT client_id FROM mamba_fact_encounter_hiv_art_card
+    WHERE cd4 < 200
+) a`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                expect(result.sql).toContain('WHERE cd4 < 200');
+            });
+
+            test('should preserve CD4 >= 200', () => {
+                const input = `SELECT COUNT(DISTINCT a.client_id)
+FROM (
+    SELECT client_id FROM mamba_fact_encounter_hiv_art_card
+    WHERE cd4 >= 200
+) a`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                expect(result.sql).toContain('WHERE cd4 >= 200');
+            });
+
+            test('should preserve CD4 IS NULL', () => {
+                const input = `SELECT COUNT(DISTINCT a.client_id)
+FROM (
+    SELECT client_id FROM mamba_fact_encounter_hiv_art_card
+    WHERE cd4 IS NULL
+) a`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                expect(result.sql).toContain('WHERE cd4 IS NULL');
+            });
+        });
+
+        describe('Patient ID normalization', () => {
+            test('should normalize client_id to patient_id', () => {
+                // Use a format that matches Pattern 1
+                const input = `SELECT COUNT(DISTINCT a.client_id) FROM (SELECT client_id FROM table_a) a`;
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                expect(result.sql).toMatch(/AS\s+patient_id/i);
+            });
+
+            test('should detect patient_id column from inner query', () => {
+                const input = `SELECT COUNT(DISTINCT a.client_id)
+FROM (
+    SELECT patient_id FROM some_table GROUP BY patient_id
+) a`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                // The interpreter normalizes to patient_id in the output SQL
+                // but tracks the source column (client_id from the COUNT expression)
+                expect(result.patientIdColumn).toBe('client_id');
+                expect(result.sql).toContain('AS patient_id');
+            });
+        });
+
+        describe('Already aggregated age/gender CUSTOM SQL', () => {
+            test('should extract population from age_group disaggregated query', () => {
+                const input = `SELECT mda.datim_agegroup AS age_group, mdp.gender AS sex, COUNT(DISTINCT a.client_id) AS value
+FROM (SELECT client_id FROM table_a) a
+GROUP BY age_group, sex`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                // Result should be patient-level, not already aggregated
+                expect(result.sql).not.toContain('age_group');
+                expect(result.sql).not.toContain('COUNT(DISTINCT');
+                expect(result.sql).toMatch(/SELECT\s+DISTINCT/i);
+            });
+
+            test('should handle hardcoded age_group values like PWIDS', () => {
+                const input = `SELECT 'PWIDS' AS age_group, 'F' AS sex, COUNT(DISTINCT a.client_id) AS value
+FROM (SELECT client_id FROM table_a) a`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                // This SQL doesn't match our standard patterns because it has:
+                // 1. Hardcoded values for age_group ('PWIDS')
+                // 2. Hardcoded value for sex ('F')
+                // 3. No GROUP BY age_group, sex at the end (Pattern 0 requirement)
+                //
+                // The interpreter should either:
+                // 1. Extract the population from the subquery
+                // 2. Fail gracefully with a meaningful error
+                if (result.success) {
+                    // If extraction succeeds, verify the population is clean
+                    expect(result.sql).toMatch(/SELECT\s+DISTINCT/i);
+                    // The result should not have the hardcoded age_group/sex values
+                    expect(result.sql).not.toMatch(/SELECT\s+'PWIDS'\s+AS\s+age_group/i);
+                } else {
+                    // If extraction fails, verify we have a meaningful error
+                    const hasError = result.warnings?.some(w => w.includes('CUSTOM_'));
+                    expect(hasError).toBe(true);
+                }
+            });
+        });
+
+        describe('Invalid extraction', () => {
+            test('should reject malformed generated identifiers', () => {
+                // This simulates the bug where ltfp_days (a calculated column) was used as a table alias
+                const input = `SELECT COUNT(DISTINCT ltfp_days.client_id)
+FROM ...`;
+
+                // The pattern should not match because ltfp_days.client_id is not valid
+                // (ltfp_days is a column alias, not a table)
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                // Should fail gracefully or not produce invalid SQL
+                if (!result.success) {
+                    // Check that at least one warning contains an error code
+                    const hasError = result.warnings?.some(w => w.includes('CUSTOM_'));
+                    expect(hasError).toBe(true);
+                }
+            });
+
+            test('should handle empty SQL', () => {
+                const result = customIndicatorInterpreter.extractPopulationSql('');
+
+                expect(result.success).toBe(false);
+                // Check that at least one warning contains the error code
+                const hasMissingError = result.warnings?.some(w => w.includes('CUSTOM_POPULATION_MISSING'));
+                expect(hasMissingError).toBe(true);
+            });
+
+            test('should handle simple COUNT query without clear structure', () => {
+                const input = 'SELECT COUNT(*) FROM table_a';
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(false);
+                // Check that at least one warning contains the error code
+                const hasAggregatedError = result.warnings?.some(w => w.includes('CUSTOM_POPULATION_AGGREGATED'));
+                expect(hasAggregatedError).toBe(true);
+            });
+        });
+
+        describe('TX-RTT style disaggregated queries', () => {
+            test('should extract population from TX-RTT style already-disaggregated query', () => {
+                // This is the actual TX-RTT pattern with age_group, sex, and COUNT(DISTINCT)
+                const input = `SELECT mda.datim_agegroup AS age_group, mdp.gender AS sex, COUNT(DISTINCT a.client_id) AS value
+FROM (SELECT client_id, TIMESTAMPDIFF(DAY, MAX(return_visit_date), :endDate) AS ltfp_days FROM mamba_fact_encounter_hiv_art_card WHERE encounter_date <= :endDate GROUP BY client_id) a
+INNER JOIN mamba_fact_patients_latest_patient_demographics mdp ON a.client_id = mdp.patient_id
+WHERE a.ltfp_days <= 28
+GROUP BY age_group, sex`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                // For now, this should either succeed with proper extraction or fail gracefully
+                // The TX-RTT pattern is complex and may need additional refinement
+                if (result.success) {
+                    // If extraction succeeds, verify the structure
+                    expect(result.sql).toMatch(/SELECT\s+DISTINCT/i);
+                    expect(result.sql).not.toContain('COUNT(DISTINCT');
+                } else {
+                    // If extraction fails, verify we have a meaningful error
+                    const hasError = result.warnings?.some(w => w.includes('CUSTOM_'));
+                    expect(hasError).toBe(true);
+                }
+            });
+        });
+    });
+
+    describe('applyDisaggregation', () => {
+        describe('Validation', () => {
+            test('should reject empty population SQL', () => {
+                const result = customIndicatorInterpreter.applyDisaggregation(
+                    '',
+                    { ageCategoryCode: 'MOH_105_OPD_DIAG', genders: ['F', 'M'] }
+                );
+
+                expect(result).toContain('CUSTOM_POPULATION_EMPTY');
+            });
+
+            test('should reject COUNT aggregate instead of population', () => {
+                const result = customIndicatorInterpreter.applyDisaggregation(
+                    'SELECT COUNT(*) FROM table_a',
+                    { ageCategoryCode: 'MOH_105_OPD_DIAG', genders: ['F', 'M'] }
+                );
+
+                expect(result).toContain('CUSTOM_POPULATION_AGGREGATED');
+            });
+
+            test('should reject SQL without patient_id column', () => {
+                const result = customIndicatorInterpreter.applyDisaggregation(
+                    'SELECT DISTINCT some_column FROM table_a',
+                    { ageCategoryCode: 'MOH_105_OPD_DIAG', genders: ['F', 'M'] }
+                );
+
+                expect(result).toContain('CUSTOM_PATIENT_ID_MISSING');
+            });
+
+            test('should reject already disaggregated SQL with age_group', () => {
+                // Create SQL that has both patient_id and age_group with AS
+                // to test that age_group validation comes first
+                const result = customIndicatorInterpreter.applyDisaggregation(
+                    'SELECT DISTINCT mda.code AS age_group, a.client_id AS patient_id FROM table_a a',
+                    { ageCategoryCode: 'MOH_105_OPD_DIAG', genders: ['F', 'M'] }
+                );
+
+                expect(result).toContain('CUSTOM_ALREADY_DISAGGREGATED');
+            });
+        });
+
+        describe('Valid disaggregation', () => {
+            test('should generate canonical disaggregation SQL', () => {
+                const populationSql = `SELECT DISTINCT a.client_id AS patient_id
+FROM (
+    SELECT client_id FROM mamba_fact_encounter_hiv_art_card
+) a
+WHERE a.some_column = 'value'`;
+
+                const result = customIndicatorInterpreter.applyDisaggregation(
+                    populationSql,
+                    { ageCategoryCode: 'MOH_105_OPD_DIAG', genders: ['F', 'M'] }
+                );
+
+                // Should generate the canonical structure
+                expect(result).toContain('WITH base_pop AS (');
+                expect(result).toContain('ag AS (');
+                expect(result).toContain('ag.age_group_id');
+                expect(result).toContain('ag.label');
+                expect(result).toContain('ag.min_age_days');
+                expect(result).toContain('ag.max_age_days');
+                expect(result).toContain("ac.code = 'MOH_105_OPD_DIAG'");
+                expect(result).toContain('COUNT(DISTINCT base_pop.patient_id) AS value');
+                expect(result).toMatch(/FROM\s+base_pop/);
+                expect(result).toMatch(/JOIN\s+mamba_fact_patients_latest_patient_demographics\s+mdp/);
+                expect(result).toContain('ON mdp.patient_id = base_pop.patient_id');
+            });
+
+            test('should use custom patient ID column from config', () => {
+                const populationSql = `SELECT DISTINCT a.client_id AS patient_id
+FROM table_a a`;
+
+                const result = customIndicatorInterpreter.applyDisaggregation(
+                    populationSql,
+                    { ageCategoryCode: 'MOH_105_OPD_DIAG', genders: ['F', 'M'] },
+                    {
+                        version: 1,
+                        patientIdColumn: 'client_id',
+                        populationQuery: { extractFrom: 'configJson' },
+                        supportsRedisaggregation: true,
+                        redisaggregationStrategy: 'population-extraction',
+                        themeIndependent: true
+                    }
+                );
+
+                // Should use the custom patient ID column
+                expect(result).toContain('ON mdp.client_id = base_pop.client_id');
+            });
+
+            test('should preserve population business logic in base_pop', () => {
+                const populationSql = `SELECT DISTINCT a.client_id AS patient_id
+FROM (
+    SELECT
+        client_id,
+        TIMESTAMPDIFF(DAY, MAX(return_visit_date), :endDate) AS ltfp_days
+    FROM mamba_fact_encounter_hiv_art_card
+    WHERE encounter_date <= :endDate
+    GROUP BY client_id
+) a
+LEFT JOIN person p ON p.person_id = a.client_id
+WHERE a.ltfp_days <= 28 AND p.person_id IS NULL`;
+
+                const result = customIndicatorInterpreter.applyDisaggregation(
+                    populationSql,
+                    { ageCategoryCode: 'MOH_105_OPD_DIAG', genders: ['F', 'M'] }
+                );
+
+                // All business logic should be preserved in base_pop
+                expect(result).toContain('TIMESTAMPDIFF(DAY, MAX(return_visit_date)');
+                expect(result).toContain('WHERE encounter_date <= :endDate');
+                expect(result).toContain('LEFT JOIN person p');
+                expect(result).toContain('WHERE a.ltfp_days <= 28');
+            });
+        });
+
+        describe('Trailing semicolon handling', () => {
+            test('should remove trailing semicolon from population SQL', () => {
+                const populationSql = `SELECT DISTINCT client_id AS patient_id FROM table_a;`;
+
+                const result = customIndicatorInterpreter.applyDisaggregation(
+                    populationSql,
+                    { ageCategoryCode: 'MOH_105_OPD_DIAG', genders: ['F', 'M'] }
+                );
+
+                // The base_pop CTE should not have a semicolon inside
+                expect(result).toMatch(/WITH base_pop AS \(\s+SELECT DISTINCT/);
+                // But the final query should have one
+                expect(result).toMatch(/ORDER BY ag\.sort_order, g\.gender;?$/);
+            });
+        });
+
+        describe('Nested subquery handling (balanced parenthesis matching)', () => {
+            test('should handle nested subqueries in inner query (TX_RTT CD4 >= 200 pattern)', () => {
+                // This is the actual bug we're fixing - nested subqueries within the inner query
+                // The inner query has LEFT JOIN with subqueries, DATE_SUB with INTERVAL, etc.
+                const input = `SELECT COUNT(DISTINCT a.client_id)
+	FROM (
+	    SELECT client_id, TIMESTAMPDIFF(DAY, MAX(return_visit_date), :endDate) ltfp_days
+	    FROM mamba_fact_encounter_hiv_art_card
+	    WHERE encounter_date <= :endDate
+	      AND return_visit_date >= :startDate
+	    GROUP BY client_id
+	) a
+	INNER JOIN mamba_fact_patients_latest_patient_demographics mdp ON a.client_id = mdp.patient_id
+	LEFT JOIN (SELECT * FROM person p WHERE p.dead = 1 AND p.death_date <= :endDate) p ON a.client_id = p.person_id
+	LEFT JOIN (
+	    SELECT mf_to.client_id
+	    FROM mamba_fact_transfer_out mf_to
+	    LEFT JOIN mamba_fact_transfer_in mf_ti ON mf_to.client_id = mf_ti.client_id
+	    WHERE transfer_out_date <= :endDate
+	      AND (transfer_out_date > transfer_in_date OR mf_ti.client_id IS NULL)
+	) mfto ON a.client_id = mfto.client_id
+	INNER JOIN (
+	    SELECT DISTINCT a.client_id
+	    FROM (
+	        SELECT client_id,
+	               TIMESTAMPDIFF(DAY, MAX(return_visit_date), DATE_SUB(:endDate, INTERVAL 3 MONTH)) ltfp_days
+	        FROM mamba_fact_encounter_hiv_art_card
+	        WHERE encounter_date <= DATE_SUB(:endDate, INTERVAL 3 MONTH)
+	          AND return_visit_date <= DATE_SUB(:endDate, INTERVAL 3 MONTH)
+	        GROUP BY client_id
+	    ) a
+	    WHERE a.ltfp_days <= 28
+	) filter_client ON a.client_id = filter_client.client_id`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                // With balanced parenthesis matching, this should now succeed
+                expect(result.success).toBe(true);
+                expect(result.sql).toMatch(/SELECT\s+DISTINCT\s+a\.client_id\s+AS\s+patient_id/i);
+
+                // Verify the complete inner query is preserved
+                expect(result.sql).toContain('TIMESTAMPDIFF(DAY, MAX(return_visit_date), :endDate)');
+                expect(result.sql).toContain('WHERE encounter_date <= :endDate');
+
+                // Verify nested subqueries are preserved
+                expect(result.sql).toContain('SELECT * FROM person p WHERE p.dead = 1');
+                expect(result.sql).toContain('SELECT mf_to.client_id');
+                expect(result.sql).toContain('SELECT DISTINCT a.client_id');
+
+                // Verify the most deeply nested subquery with DATE_SUB is preserved
+                expect(result.sql).toContain('DATE_SUB(:endDate, INTERVAL 3 MONTH)');
+            });
+
+            test('should handle deeply nested subqueries with INTERVAL expressions', () => {
+                // Test with multiple levels of nesting and INTERVAL expressions
+                const input = `SELECT COUNT(DISTINCT base.client_id)
+	FROM (
+	    SELECT DISTINCT a.client_id
+	    FROM (
+	        SELECT client_id
+	        FROM mamba_fact_encounter_hiv_art_card
+	        WHERE encounter_date <= DATE_SUB(:endDate, INTERVAL 3 MONTH)
+	        GROUP BY client_id
+	    ) a
+	    INNER JOIN (
+	        SELECT client_id, cd4_result
+	        FROM mamba_fact_lab_results
+	        WHERE test_date >= DATE_SUB(:endDate, INTERVAL 3 MONTH)
+	    ) lab ON a.client_id = lab.client_id
+	) base`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                // Verify all levels of nesting are preserved
+                expect(result.sql).toContain('DATE_SUB(:endDate, INTERVAL 3 MONTH)');
+                expect(result.sql).toContain('mamba_fact_encounter_hiv_art_card');
+                expect(result.sql).toContain('mamba_fact_lab_results');
+            });
+
+            test('should handle TX_ML pattern with quoted parameters and extra column', () => {
+                // This is the TX_ML pattern with:
+                // 1. Quoted parameters like ':endDate' instead of :endDate
+                // 2. Extra column before COUNT: SELECT 'PWIDS', COUNT(DISTINCT a.client_id)
+                const input = `SELECT 'PWIDS', COUNT(DISTINCT a.client_id)
+FROM (SELECT a.client_id, return_date
+      FROM (SELECT client_id, MAX(return_visit_date) return_date
+            FROM mamba_fact_encounter_hiv_art_card
+            WHERE encounter_date <= ':endDate'
+              AND return_visit_date >= DATE_SUB(':startDate', INTERVAL 3 MONTH)
+            GROUP BY client_id) a
+      WHERE TIMESTAMPDIFF(DAY, return_date, ':endDate') > 28
+        AND TIMESTAMPDIFF(DAY, return_date, DATE_SUB(':endDate', INTERVAL 3 MONTH)) <= 28) a
+         INNER JOIN mamba_fact_patients_latest_patient_demographics mdp ON a.client_id = mdp.patient_id
+         INNER JOIN (SELECT client_id
+                     FROM mamba_fact_encounter_hiv_art_summary
+                     WHERE special_category = 'Current drug user') special_category
+                    ON a.client_id = special_category.client_id`;
+
+                const result = customIndicatorInterpreter.extractPopulationSql(input);
+
+                expect(result.success).toBe(true);
+                expect(result.sql).toMatch(/SELECT\s+DISTINCT\s+a\.client_id\s+AS\s+patient_id/i);
+
+                // Verify quoted parameters are converted to unquoted
+                expect(result.sql).toMatch(/:endDate/);
+                expect(result.sql).not.toMatch(/':endDate'/);
+
+                // Verify the special_category filter is preserved
+                expect(result.sql).toContain('special_category = \'Current drug user\'');
+            });
+        });
+    });
+});