@openmrs/esm-report-builder 0.1.1-pre.87

package/src/components/indicators/utils/final-indicator-sql.utils.ts

@@ -0,0 +1,190 @@
+import type { IndicatorDto } from '../../../resources/indicator/indicators.api';
+
+import {
+    countSqlToPopulationSql,
+    tryGetPatientIdColumnFromConfig,
+    tryGetCountSqlFromIndicator,
+} from './composite-indicator-sql.utils';
+import {
+    compilePopulationSql,
+    generateAgeSexDisaggregationSql,
+    clearCompilationCache,
+    type CompilerOptions
+} from './population-sql.compiler';
+
+export type FinalIndicatorAuthoringV1 = {
+    version: 1;
+    baseIndicatorId: string;
+
+    // Backward/forward compatibility
+    ageGroupSetCode?: string;
+    ageCategoryCode?: string;
+
+    genders: Array<'F' | 'M'>;
+    sqlPreview: string;
+};
+
+type BuildFinalSqlArgs = {
+    baseIndicator: IndicatorDto;
+    ageCategoryCode: string;
+    genders: Array<'F' | 'M'>;
+};
+
+type BuildFinalSqlAsyncArgs = BuildFinalSqlArgs & {
+    getIndicator: (uuid: string) => Promise<IndicatorDto | null>;
+    compilerOptions?: CompilerOptions;
+};
+
+/**
+ * Builds a Final Indicator query that ALWAYS returns all age groups (even if value = 0),
+ * and for each selected gender.
+ *
+ * Strategy:
+ *  - base_pop: patient population set from base indicator
+ *  - ag: all age groups for the selected category (active only)
+ *  - genders: the selected genders as rows
+ *  - cnt: computed counts per (age_group_id, gender)
+ *  - final: ag CROSS JOIN genders LEFT JOIN cnt, COALESCE(value,0)
+ */
+export function buildFinalIndicatorSql({ baseIndicator, ageCategoryCode, genders }: BuildFinalSqlArgs): string {
+    const { sql: countSql } = tryGetCountSqlFromIndicator(baseIndicator);
+    const pidCol = tryGetPatientIdColumnFromConfig(baseIndicator);
+
+    const populationSql = countSql ? countSqlToPopulationSql(countSql, pidCol, 'Patients') : '';
+
+    if (!populationSql.trim()) {
+        return `-- Unable to build final indicator SQL: base indicator does not contain usable count SQL.\n-- Ensure the base indicator has a valid sqlTemplate or configJson.sqlPreview.`;
+    }
+
+    // Default to both genders if none selected (for backward compatibility)
+    // TODO: Remove this default in a future version and require explicit gender selection
+    const selectedGenders = (genders ?? []).length ? genders : (['F', 'M'] as Array<'F' | 'M'>);
+
+    // Log deprecation warning when defaulting
+    if (!genders || genders.length === 0) {
+        console.warn('[buildFinalIndicatorSql] No genders specified, defaulting to both F and M. This default will be removed in a future version.');
+    }
+
+    const gendersCte = buildGenderCte(selectedGenders);
+
+    // NOTE: adjust these table names if your DB uses different names.
+    // If you adopted the new naming, use report_builder_dim_age_category/group.
+    const AGE_CATEGORY_TABLE = 'report_builder_dim_age_category';
+    const AGE_GROUP_TABLE = 'report_builder_dim_age_group';
+
+    return `
+WITH base_pop AS (
+${indent(populationSql.trim(), 2)}
+),
+ag AS (
+  SELECT
+    ag.age_group_id,
+    ag.label,
+    ag.min_age_days,
+    ag.max_age_days,
+    ag.sort_order
+  FROM ${AGE_GROUP_TABLE} ag
+  JOIN ${AGE_CATEGORY_TABLE} ac
+    ON ac.age_category_id = ag.age_category_id
+  WHERE ac.code = '${escapeSqlLiteral(ageCategoryCode)}'
+    AND ag.is_active = 1
+),
+genders AS (
+${indent(gendersCte, 2)}
+),
+cnt AS (
+  SELECT
+    ag.age_group_id AS age_group_id,
+    mdp.gender AS gender,
+    COUNT(DISTINCT base_pop.${pidCol}) AS value
+  FROM base_pop
+  JOIN mamba_fact_patients_latest_patient_demographics mdp
+    ON mdp.patient_id = base_pop.${pidCol}
+  JOIN ag
+    ON TIMESTAMPDIFF(DAY, mdp.birthdate, :endDate)
+       BETWEEN ag.min_age_days AND ag.max_age_days
+  WHERE mdp.birthdate IS NOT NULL
+    AND mdp.gender IS NOT NULL
+    AND mdp.gender IN (${selectedGenders.map((g) => `'${g}'`).join(',')})
+  GROUP BY ag.age_group_id, mdp.gender
+)
+SELECT
+  ag.label AS age_group,
+  g.gender AS gender,
+  COALESCE(cnt.value, 0) AS value
+FROM ag
+CROSS JOIN genders g
+LEFT JOIN cnt
+  ON cnt.age_group_id = ag.age_group_id
+ AND cnt.gender = g.gender
+ORDER BY ag.sort_order, g.gender;
+`.trim();
+}
+
+function buildGenderCte(genders: Array<'F' | 'M'>): string {
+    // Produce rows:
+    // SELECT 'F' AS gender UNION ALL SELECT 'M' AS gender
+    const parts = genders.map((g, i) => (i === 0 ? `SELECT '${g}' AS gender` : `UNION ALL SELECT '${g}' AS gender`));
+    return parts.join('\n');
+}
+
+function indent(s: string, spaces: number) {
+    const pad = ' '.repeat(spaces);
+    return s
+        .split('\n')
+        .map((l) => (l.trim().length ? pad + l : l))
+        .join('\n');
+}
+
+function escapeSqlLiteral(s: string) {
+    return String(s).replace(/'/g, "''");
+}
+
+/**
+ * Async version of final indicator SQL builder.
+ *
+ * This function uses the recursive population SQL compiler to properly handle
+ * composite base indicators. For simple base indicators, it falls back to the
+ * synchronous logic.
+ *
+ * Use this version when the base indicator might be a composite indicator.
+ */
+export async function buildFinalIndicatorSqlAsync({
+    baseIndicator,
+    ageCategoryCode,
+    genders,
+    getIndicator,
+    compilerOptions = {}
+}: BuildFinalSqlAsyncArgs): Promise<string> {
+    try {
+        // Clear cache to ensure fresh compilation
+        clearCompilationCache();
+
+        // If base indicator is composite, use the recursive compiler
+        if (baseIndicator.kind === 'COMPOSITE') {
+            const result = await compilePopulationSql(baseIndicator, getIndicator, new Set(), compilerOptions);
+
+            // Check for compilation errors
+            if (result.error) {
+                return `-- Error: ${result.error.message}\n-- Base indicator: ${baseIndicator.name} (${baseIndicator.code || baseIndicator.uuid})\n-- Error Code: ${result.error.code}`;
+            }
+
+            const populationSql = result.sql;
+
+            // Generate the disaggregation SQL using the compiled population SQL
+            // Note: population SQL is normalized to patient_id, so no need to specify patientIdColumn
+            return generateAgeSexDisaggregationSql({
+                populationSql,
+                ageCategoryCode,
+                genders
+            });
+        }
+
+        // For BASE indicators, use the synchronous logic
+        return buildFinalIndicatorSql({ baseIndicator, ageCategoryCode, genders });
+    } catch (error) {
+        // Return an error comment in the SQL
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        return `-- Error building final indicator SQL: ${errorMsg}\n-- Base indicator: ${baseIndicator.name} (${baseIndicator.code || baseIndicator.uuid})`;
+    }
+}

package/src/components/indicators/utils/indicator-conditions-hydration.utils.ts

@@ -0,0 +1,254 @@
+// src/report-builder/components/indicators/utils/indicator-conditions-hydration.utils.ts
+
+import type { ThemeCondition } from '../types/data-theme-config.types';
+import type { IndicatorCondition } from '../types/indicator-types';
+
+import { getConceptByUuid } from '../../../resources/concepts/concepts.resource';
+import type { ConceptSummary } from '../../../resources/concepts/concept-types';
+
+import { toSelectedConcept, type SelectedConcept } from '../handler/concept-search-multiselect.component';
+import type { ConceptUiMap, QaUiMap } from '../types/condition-ui.types';
+
+export type HydrationOptions = {
+    force?: boolean;
+    dedupe?: boolean;
+    preserveUnknownKeys?: boolean;
+};
+
+export type HydrationResult = {
+    conceptUi: ConceptUiMap;
+    qaUi: QaUiMap;
+    stats: {
+        themeConditionsCount: number;
+        pickedConditionsCount: number;
+        conceptConditionsHydrated: number;
+        qaConditionsHydrated: number;
+        tokensHydrated: number;
+        tokensMissed: number;
+    };
+};
+
+// Stored value shape in pickedConditions for QA handler
+// (support both legacy single-question and current multi-question formats)
+type QAValue = {
+    question?: string | number | null;
+    questions?: Array<string | number>;
+    answers?: Array<string | number>;
+};
+
+function uniq(tokens: string[]) {
+    const out: string[] = [];
+    const seen = new Set<string>();
+    for (const t of tokens) {
+        const k = String(t ?? '').trim();
+        if (!k) continue;
+        if (seen.has(k)) continue;
+        seen.add(k);
+        out.push(k);
+    }
+    return out;
+}
+
+/**
+ * Hydrate ONE token into ConceptSummary (internal).
+ * We now persist UUIDs, so we call direct endpoint by UUID.
+ */
+async function hydrateConceptSummaryToken(token: string, signal?: AbortSignal): Promise<ConceptSummary | null> {
+    const tok = String(token ?? '').trim();
+    if (!tok) return null;
+
+    try {
+        const concept = await getConceptByUuid(tok, signal);
+        if (!concept?.uuid) return null;
+        return concept;
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * Hydrate ONE token into SelectedConcept (UI shape).
+ * ✅ Uses toSelectedConcept so ICD mappings are preserved.
+ */
+async function hydrateConceptToken(token: string, signal?: AbortSignal): Promise<SelectedConcept | null> {
+    const cs = await hydrateConceptSummaryToken(token, signal);
+    return cs ? toSelectedConcept(cs) : null;
+}
+
+/**
+ * Hydrate MANY tokens into SelectedConcept[].
+ * ✅ for-loop; preserves order; returns only valid concepts.
+ */
+async function hydrateConceptTokens(tokens: string[], signal?: AbortSignal, dedupe = true): Promise<SelectedConcept[]> {
+    const clean = dedupe ? uniq(tokens) : tokens.map((t) => String(t ?? '').trim()).filter(Boolean);
+    if (!clean.length) return [];
+
+    const results: SelectedConcept[] = [];
+
+    for (const tok of clean) {
+        if (signal?.aborted) break;
+
+        try {
+            const concept = await hydrateConceptToken(tok, signal);
+            if (concept) results.push(concept);
+        } catch {
+            // ignore individual token failures
+        }
+    }
+
+    return results;
+}
+
+/**
+ * Hydrates UI state for concept-based conditions (edit mode):
+ * - iterate picked first (source of truth)
+ * - match to themeByKey for handler
+ * - ensure defaults for theme conditions not saved
+ */
+export async function hydrateConditionUiState(
+    themeConditions: ThemeCondition[] | undefined,
+    pickedConditions: IndicatorCondition[] | undefined,
+    existingConceptUi: ConceptUiMap | undefined,
+    existingQaUi: QaUiMap | undefined,
+    signal?: AbortSignal,
+    options?: HydrationOptions,
+): Promise<HydrationResult> {
+    const opt: Required<HydrationOptions> = {
+        force: options?.force ?? true,
+        dedupe: options?.dedupe ?? true,
+        preserveUnknownKeys: options?.preserveUnknownKeys ?? true,
+    };
+
+    const themeList = themeConditions ?? [];
+    const pickedList = pickedConditions ?? [];
+
+    const themeByKey = new Map<string, ThemeCondition>();
+    for (const tc of themeList) {
+        if (tc?.key) themeByKey.set(tc.key, tc);
+    }
+
+    const nextConceptUi: ConceptUiMap = { ...(existingConceptUi ?? {}) };
+    const nextQaUi: QaUiMap = { ...(existingQaUi ?? {}) };
+
+    let conceptConditionsHydrated = 0;
+    let qaConditionsHydrated = 0;
+    let tokensHydrated = 0;
+    let tokensMissed = 0;
+
+    const seenKeys = new Set<string>();
+
+    // 1) hydrate from picked conditions (source of truth)
+    for (const pc of pickedList) {
+        if (!pc?.key) continue;
+        seenKeys.add(pc.key);
+
+        const tc = themeByKey.get(pc.key);
+        if (!tc) {
+            if (!opt.preserveUnknownKeys) {
+                delete nextConceptUi[pc.key];
+                delete nextQaUi[pc.key];
+            }
+            continue;
+        }
+
+        // -----------------------------
+        // CONCEPT_SEARCH
+        // -----------------------------
+        if (tc.handler === 'CONCEPT_SEARCH') {
+            const shouldHydrate =
+                opt.force || !Array.isArray(nextConceptUi[tc.key]) || nextConceptUi[tc.key].length === 0;
+
+            if (!shouldHydrate) continue;
+
+            const raw = pc.value;
+
+            if (Array.isArray(raw) && raw.length) {
+                const tokens = raw.map((x) => String(x)).filter(Boolean);
+                const selected = await hydrateConceptTokens(tokens, signal, opt.dedupe);
+
+                tokensHydrated += selected.length;
+                tokensMissed += Math.max(0, tokens.length - selected.length);
+
+                nextConceptUi[tc.key] = selected;
+            } else {
+                nextConceptUi[tc.key] = [];
+            }
+
+            conceptConditionsHydrated += 1;
+            continue;
+        }
+
+        // -----------------------------
+        // QUESTION_ANSWER_CONCEPT_SEARCH
+        // -----------------------------
+        if (tc.handler === 'QUESTION_ANSWER_CONCEPT_SEARCH') {
+            const existing = nextQaUi[tc.key];
+
+            const shouldHydrate =
+                opt.force ||
+                !existing ||
+                ((existing.questions?.length ?? 0) === 0 && (existing.answers?.length ?? 0) === 0 && !existing.error);
+
+            if (!shouldHydrate) continue;
+
+            const raw: any = pc.value;
+
+            const qa: QAValue | null = raw && typeof raw === 'object' ? (raw as QAValue) : null;
+
+            // ✅ support both shapes: { question } OR { questions: [] }
+            const qTokens: string[] = Array.isArray(qa?.questions)
+                ? (qa?.questions ?? []).map((x) => String(x).trim()).filter(Boolean)
+                : qa?.question !== null && qa?.question !== undefined
+                    ? [String(qa.question).trim()].filter(Boolean)
+                    : [];
+
+            const aTokens: string[] = Array.isArray(qa?.answers)
+                ? (qa?.answers ?? []).map((x) => String(x).trim()).filter(Boolean)
+                : [];
+
+            const questions: SelectedConcept[] = qTokens.length
+                ? await hydrateConceptTokens(qTokens, signal, opt.dedupe)
+                : [];
+
+            const answers: SelectedConcept[] = aTokens.length
+                ? await hydrateConceptTokens(aTokens, signal, opt.dedupe)
+                : [];
+
+            tokensHydrated += questions.length + answers.length;
+            tokensMissed += Math.max(0, qTokens.length - questions.length) + Math.max(0, aTokens.length - answers.length);
+
+            nextQaUi[tc.key] = { questions, answers };
+            qaConditionsHydrated += 1;
+            continue;
+        }
+
+        // other handlers ignored
+    }
+
+    // 2) ensure all theme conditions have UI defaults
+    for (const tc of themeList) {
+        if (!tc?.key) continue;
+
+        if (!seenKeys.has(tc.key)) {
+            if (tc.handler === 'CONCEPT_SEARCH') {
+                if (!Array.isArray(nextConceptUi[tc.key])) nextConceptUi[tc.key] = [];
+            }
+            if (tc.handler === 'QUESTION_ANSWER_CONCEPT_SEARCH') {
+                if (!nextQaUi[tc.key]) nextQaUi[tc.key] = { questions: [], answers: [] };
+            }
+        }
+    }
+
+    return {
+        conceptUi: nextConceptUi,
+        qaUi: nextQaUi,
+        stats: {
+            themeConditionsCount: themeList.length,
+            pickedConditionsCount: pickedList.length,
+            conceptConditionsHydrated,
+            qaConditionsHydrated,
+            tokensHydrated,
+            tokensMissed,
+        },
+    };
+}

package/src/components/indicators/utils/indicator-sql-resolution.utils.ts

@@ -0,0 +1,204 @@
+/**
+ * Centralized Indicator SQL Resolution
+ *
+ * This module provides a single canonical way to resolve SQL from indicators
+ * across all compilation paths (Indicator Preview, Section Preview, etc.).
+ *
+ * Instead of duplicating the fallback chain in multiple components,
+ * use these functions to reliably locate and extract SQL from indicator configurations.
+ */
+
+import type { IndicatorDto } from '../../../resources/indicator/indicators.api';
+
+/**
+ * SQL extraction source for debugging and error reporting
+ */
+export type SqlSource =
+    | 'indicator.sqlTemplate'
+    | 'configJson.sqlPreview'
+    | 'configJson.sqlTemplate'
+    | 'configJson.base.sqlPreview'
+    | 'configJson.base.sqlTemplate'
+    | 'configJson.authoring.base.sqlPreview'
+    | 'configJson.authoring.base.sqlTemplate'
+    | 'configJson.baseIndicator.sqlPreview'
+    | 'configJson.baseIndicator.sqlTemplate'
+    | 'none'
+    | 'none(parse-error)'
+    | 'none(empty-config)';
+
+/**
+ * Result of resolving indicator SQL with source tracking for debugging
+ */
+export type ResolvedIndicatorSql = {
+    /** The extracted SQL */
+    sql: string;
+    /** Where the SQL was found (for debugging) */
+    source: SqlSource;
+    /** The config JSON that was parsed (for reference) */
+    parsedConfig?: any;
+};
+
+/**
+ * Structured error when SQL cannot be resolved
+ */
+export type SqlResolutionError = {
+    success: false;
+    error: string;
+    indicator: {
+        uuid: string;
+        code?: string;
+        name?: string;
+        kind?: string;
+    };
+    attemptedSources: SqlSource[];
+};
+
+/**
+ * Resolve SQL from an indicator with comprehensive fallback chain.
+ *
+ * This is the canonical way to extract SQL from indicators across all compilation paths.
+ *
+ * Priority order (as per the current config structure):
+ * 1. indicator.sqlTemplate (direct field)
+ * 2. configJson.sqlPreview (flat config)
+ * 3. configJson.sqlTemplate (flat config)
+ * 4. configJson.base.sqlPreview (nested base config)
+ * 5. configJson.base.sqlTemplate (nested base config)
+ * 6. configJson.authoring.base.sqlPreview (authoring structure)
+ * 7. configJson.authoring.base.sqlTemplate (authoring structure)
+ * 8. configJson.baseIndicator.sqlPreview (wrapped indicator config)
+ * 9. configJson.baseIndicator.sqlTemplate (wrapped indicator config)
+ *
+ * @param indicator - The indicator to extract SQL from
+ * @returns Resolved SQL with source tracking, or structured error
+ */
+export function resolveIndicatorSql(indicator: IndicatorDto): ResolvedIndicatorSql | SqlResolutionError {
+    const attemptedSources: SqlSource[] = [];
+    const direct = (indicator?.sqlTemplate ?? '').trim();
+
+    if (direct) {
+        return {
+            sql: normalizeEscapedNewlines(direct),
+            source: 'indicator.sqlTemplate'
+        };
+    }
+    attemptedSources.push('indicator.sqlTemplate');
+
+    // Try to parse config JSON
+    let parsed: any = null;
+    try {
+        parsed = indicator?.configJson ? JSON.parse(indicator.configJson) : null;
+    } catch (e) {
+        // Invalid JSON - will return error at end
+        parsed = null;
+    }
+
+    if (!parsed) {
+        return createSqlResolutionError(indicator, attemptedSources, 'configJson is null or invalid JSON');
+    }
+
+    // Helper to try a path and return early if found
+    const tryPath = (source: SqlSource, getValue: (p: any) => string): ResolvedIndicatorSql | null => {
+        attemptedSources.push(source);
+        const value = (getValue(parsed) ?? '').trim();
+        if (value) {
+            return {
+                sql: normalizeEscapedNewlines(value),
+                source,
+                parsedConfig: parsed
+            };
+        }
+        return null;
+    };
+
+    // Try all known paths in priority order
+    const result =
+        tryPath('configJson.sqlPreview', (p) => p.sqlPreview) ||
+        tryPath('configJson.sqlTemplate', (p) => p.sqlTemplate) ||
+        tryPath('configJson.base.sqlPreview', (p) => p.base?.sqlPreview) ||
+        tryPath('configJson.base.sqlTemplate', (p) => p.base?.sqlTemplate) ||
+        tryPath('configJson.authoring.base.sqlPreview', (p) => p.authoring?.base?.sqlPreview) ||
+        tryPath('configJson.authoring.base.sqlTemplate', (p) => p.authoring?.base?.sqlTemplate) ||
+        tryPath('configJson.baseIndicator.sqlPreview', (p) => p.baseIndicator?.sqlPreview) ||
+        tryPath('configJson.baseIndicator.sqlTemplate', (p) => p.baseIndicator?.sqlTemplate);
+
+    if (result) {
+        return result;
+    }
+
+    attemptedSources.push('none');
+    return createSqlResolutionError(indicator, attemptedSources, 'No SQL found in any known config location');
+}
+
+/**
+ * Create a structured SQL resolution error
+ */
+function createSqlResolutionError(
+    indicator: IndicatorDto,
+    attemptedSources: SqlSource[],
+    reason: string
+): SqlResolutionError {
+    return {
+        success: false,
+        error: `SQL_RESOLUTION_FAILED: ${reason}`,
+        indicator: {
+            uuid: indicator.uuid,
+            code: indicator.code,
+            name: indicator.name,
+            kind: indicator.kind
+        },
+        attemptedSources
+    };
+}
+
+/**
+ * Check if a resolved SQL result is an error
+ */
+export function isSqlResolutionError(result: ResolvedIndicatorSql | SqlResolutionError): result is SqlResolutionError {
+    return (result as any).success === false;
+}
+
+/**
+ * Resolve patient ID column from indicator config.
+ *
+ * Centralized way to extract the patient ID column across all compilation paths.
+ *
+ * Priority order:
+ * 1. parsed.themeConfig.patientIdColumn
+ * 2. parsed.base.themeConfig.patientIdColumn
+ * 3. parsed.authoring.base.themeConfig.patientIdColumn
+ * 4. parsed.baseIndicator.themeConfig.patientIdColumn
+ * 5. Default: 'client_id'
+ *
+ * @param indicator - The indicator to extract patient ID column from
+ * @returns The patient ID column name (defaults to 'client_id')
+ */
+export function resolvePatientIdColumn(indicator: IndicatorDto): string {
+    try {
+        const parsed: any = indicator?.configJson ? JSON.parse(indicator.configJson) : null;
+
+        const cfg =
+            parsed?.themeConfig ? { ...parsed.themeConfig, _source: 'themeConfig' } :
+            parsed?.base?.themeConfig ? { ...parsed.base.themeConfig, _source: 'base.themeConfig' } :
+            parsed?.authoring?.base?.themeConfig ? { ...parsed.authoring.base.themeConfig, _source: 'authoring.base.themeConfig' } :
+            parsed?.baseIndicator?.themeConfig ? { ...parsed.baseIndicator.themeConfig, _source: 'baseIndicator.themeConfig' } :
+            null;
+
+        const pid = cfg?.patientIdColumn;
+        return pid ? String(pid) : 'client_id';
+    } catch {
+        return 'client_id';
+    }
+}
+
+/**
+ * Normalize escaped newlines in SQL.
+ *
+ * SQL stored in JSON may have literal \n instead of actual newlines.
+ * This converts them back.
+ */
+export function normalizeEscapedNewlines(sql: string): string {
+    if (!sql) return sql;
+    return sql.replace(/\\n/g, '\n');
+}