@peopl-health/nexus 6.0.0-dev.619 → 6.0.0-dev.634

package/lib/clinical/config/divergenceConfig.js

@@ -46,7 +46,23 @@ async function load() {
   const stochastic = { ...STOCHASTIC_DEFAULTS, ...(parsedStochastic || {}) };
   if (stochastic.enabled && !stochastic.presetId) throw new Error('divergence config STOCHASTIC.presetId is required when enabled');
 
-  const config = { FIELDS: fields, ORACLE_CONNECTION: oracleConnection, STOCHASTIC: stochastic };
+  let safetyFlagPatterns = [];
+  if (byKey.SAFETY_FLAG_PATTERNS) {
+    const parsedPatterns = safeParseArray(byKey.SAFETY_FLAG_PATTERNS);
+    if (!parsedPatterns) throw new Error('divergence config SAFETY_FLAG_PATTERNS is not a JSON array');
+    safetyFlagPatterns = parsedPatterns.map((row) => {
+      if (!isPlainObject(row)) throw new Error('divergence config SAFETY_FLAG_PATTERNS row is not an object');
+      const { flag, pattern } = row;
+      if (!flag || !pattern) throw new Error('divergence config SAFETY_FLAG_PATTERNS row missing flag or pattern');
+      try {
+        return { flag, re: new RegExp(pattern, 'i') };
+      } catch (err) {
+        throw new Error(`divergence config SAFETY_FLAG_PATTERNS pattern for ${flag} is not a valid regex: ${err.message}`);
+      }
+    });
+  }
+
+  const config = { FIELDS: fields, ORACLE_CONNECTION: oracleConnection, STOCHASTIC: stochastic, SAFETY_FLAG_PATTERNS: safetyFlagPatterns };
   cache.set(CACHE_KEY, config);
   return config;
 }
@@ -60,4 +76,5 @@ module.exports = {
   getFields: () => get('FIELDS'),
   getOracleConnection: () => get('ORACLE_CONNECTION'),
   getStochastic: () => get('STOCHASTIC'),
+  getSafetyFlagPatterns: () => get('SAFETY_FLAG_PATTERNS'),
 };

package/lib/clinical/config/subAgentsConfig.js

@@ -0,0 +1,38 @@
+const { Config_ID } = require('../../config/airtableConfig');
+
+const { getRecordByFilter } = require('../../services/airtableService');
+
+const MapCache = require('../../utils/MapCache');
+const { safeParse, isPlainObject } = require('../../utils/jsonUtils');
+
+const SUBAGENTS_CONFIG_TABLE = 'sub_agents';
+const CACHE_TTL = 5 * 60 * 1000;
+const CACHE_KEY = 'subAgentsConfig';
+const EXTRACTOR_DEFAULTS = { presetId: '' };
+
+const cache = new MapCache({ maxSize: 1, ttl: CACHE_TTL });
+
+async function load() {
+  const cached = cache.get(CACHE_KEY);
+  if (cached) return cached;
+
+  const records = await getRecordByFilter(Config_ID, SUBAGENTS_CONFIG_TABLE, 'TRUE()');
+  if (!records) throw new Error('subAgents config could not be read from the Config base');
+  const byKey = Object.fromEntries(records.map((r) => [r.config, r.value]));
+
+  const parsedExtractor = safeParse(byKey.EXTRACTOR, null);
+  if (byKey.EXTRACTOR && !isPlainObject(parsedExtractor)) throw new Error('subAgents config EXTRACTOR is not a JSON object');
+  const extractor = { ...EXTRACTOR_DEFAULTS, ...(parsedExtractor || {}) };
+
+  const config = { EXTRACTOR: extractor };
+  cache.set(CACHE_KEY, config);
+  return config;
+}
+
+async function get(key) {
+  return (await load())[key];
+}
+
+module.exports = {
+  getExtractor: () => get('EXTRACTOR'),
+};

package/lib/clinical/helpers/clinicalMentionHelper.js

@@ -0,0 +1,68 @@
+const { ClinicalMention } = require('../../shared/dtos/ClinicalMention');
+
+function buildExtractorInput(text, detectedTags, curated, template) {
+  const payload = JSON.stringify({ raw_message: text, detected_tags: detectedTags }, null, 2);
+  const tpl = curated.length ? template.symptom : template.default;
+  return tpl
+    .replace('{{curated}}', () => JSON.stringify(curated, null, 2))
+    .replace('{{payload}}', () => payload);
+}
+
+function readOutputText(result) {
+  if (result?.output_text) return result.output_text;
+  const output = Array.isArray(result?.output) ? result.output : [];
+  const message = output.find((item) => item.type === 'message');
+  const content = Array.isArray(message?.content) ? message.content : [];
+  const part = content.find((c) => typeof c?.text === 'string');
+  return part?.text || '';
+}
+
+function toClinicalMention(raw, { code, turnId, index, now, intakeSource, responseFormat, spanishByTerm = {} }) {
+  const rawTerm = (raw.ctcae_term || '').trim().toLowerCase();
+  const ctcaeTerm = !rawTerm || rawTerm === 'other' ? null : rawTerm;
+  const canonicalTerm = deriveCanonicalTerm(raw, ctcaeTerm, spanishByTerm);
+  if (!canonicalTerm) return null;
+
+  const mentionId = `men_${turnId}_${index}`;
+  const verbatim = raw.verbatim_quote ? [raw.verbatim_quote] : (raw.verbatim_quotes || []);
+  const observed = raw.observed_fields || {};
+  return new ClinicalMention({
+    fact: {
+      mentionId,
+      patientId: code,
+      category: raw.category,
+      canonicalTerm,
+      reportedAt: now,
+      assertion: raw.assertion || {},
+      ctcaeTerm,
+      temporality: raw.temporality || null,
+      observedFields: responseFormat === 'router'
+        ? Object.fromEntries(Object.entries(observed).filter(([, v]) => v != null))
+        : observed,
+    },
+    provenance: {
+      mentionId,
+      patientId: code,
+      turnId,
+      recordedAt: now,
+      source: intakeSource || 'self',
+      verbatimQuotes: verbatim,
+      extractorConfidence: raw.extractor_confidence ?? raw.confidence ?? raw.assertion?.confidence ?? 1,
+    },
+  });
+}
+
+function deriveCanonicalTerm(raw, ctcaeTerm, spanishByTerm) {
+  const emitted = (raw.canonical_term || '').trim().toLowerCase();
+  if (emitted) return emitted;
+  if (raw.category === 'symptom' && ctcaeTerm) {
+    return ((spanishByTerm[ctcaeTerm] || ctcaeTerm) || '').toLowerCase() || null;
+  }
+  return null;
+}
+
+module.exports = {
+  buildExtractorInput,
+  readOutputText,
+  toClinicalMention,
+};

package/lib/clinical/helpers/safetyFlagsHelper.js

@@ -0,0 +1,10 @@
+function detectSafetyFlags(text, catalog) {
+  const flags = new Set();
+  if (!text) return [];
+  for (const { flag, re } of catalog) {
+    if (re.test(text)) flags.add(flag);
+  }
+  return [...flags];
+}
+
+module.exports = { detectSafetyFlags };

package/lib/clinical/services/clinicalExtractionService.js

@@ -0,0 +1,110 @@
+const { getSafetyFlagPatterns } = require('../config/divergenceConfig');
+const { getExtractor } = require('../config/subAgentsConfig');
+const { getCtcaeCatalog } = require('../stores/CtcaeCatalog');
+const { storeClinicalMentions } = require('../../fhir');
+
+const { Config_ID } = require('../../config/airtableConfig');
+const { getRecordByFilter } = require('../../services/airtableService');
+const MapCache = require('../../utils/MapCache');
+const { safeParse, safeParseArray } = require('../../utils/jsonUtils');
+const { logger } = require('../../utils/logger');
+const { detectSafetyFlags } = require('../helpers/safetyFlagsHelper');
+const { buildExtractorInput, readOutputText, toClinicalMention } = require('../helpers/clinicalMentionHelper');
+
+const CONFIG_TABLE = 'registry';
+const CACHE_TTL = 5 * 60 * 1000;
+const CONFIG_CACHE_KEY = 'clinicalExtractorConfig';
+
+const extractorConfigCache = new MapCache({ maxSize: 1, ttl: CACHE_TTL });
+
+async function extractClinicalMentions({ text, turnId, code, provider, detectedTags = [], intakeSource = 'self', responseFormat = 'router' }) {
+  const [patterns, { presetId }] = await Promise.all([
+    getSafetyFlagPatterns(),
+    getExtractor(),
+  ]);
+
+  const flags = new Set(detectSafetyFlags(text || '', patterns));
+
+  if (!text || !text.trim() || !presetId || !provider?.runStructured) {
+    return { clinicalMentions: [], safetyFlags: [...flags] };
+  }
+
+  try {
+    const template = await getExtractorInputTemplate();
+    const curated = detectedTags.includes('symptom') ? getCtcaeCatalog().getCuratedSymptomsByTerm() : new Map();
+    const curatedRows = [...curated.values()].map(toExtractorRow);
+    const spanishByTerm = Object.fromEntries(curatedRows.filter((row) => row.ctcae_term).map((row) => [row.ctcae_term, row.spanish_label]));
+
+    const result = await provider.runStructured({
+      presetId,
+      input: [{ role: 'user', content: buildExtractorInput(text, detectedTags, curatedRows, template) }],
+      metadata: { extractor: 'clinical-mention', turnId },
+    });
+
+    const outputText = readOutputText(result);
+    const parsed = safeParse(outputText, null);
+    if (outputText && outputText.trim() && parsed === null) {
+      logger.warn('[clinicalExtraction] model output did not parse as JSON; no mentions extracted', { turnId });
+    }
+    const rawMentions = Array.isArray(parsed?.mentions) ? parsed.mentions : [];
+
+    const now = new Date().toISOString();
+    const clinicalMentions = [];
+    rawMentions.forEach((raw, index) => {
+      if (!raw || typeof raw !== 'object') return;
+      const mention = toClinicalMention(raw, { code, turnId, index, now, intakeSource, responseFormat, spanishByTerm });
+      if (mention) clinicalMentions.push(mention);
+      for (const f of raw.safety_flags || []) flags.add(f);
+    });
+
+    const patientMentions = clinicalMentions.filter((mention) => mention.fact.assertion.experiencer === 'patient');
+    if (patientMentions.length) {
+      await storeClinicalMentions({ patientId: code, mentions: patientMentions });
+    }
+
+    return { clinicalMentions, safetyFlags: [...flags] };
+  } catch (err) {
+    logger.error('[clinicalExtraction] extraction failed; returning deterministic safety flags only', { turnId, error: err?.message });
+    return { clinicalMentions: [], safetyFlags: [...flags] };
+  }
+}
+
+function toExtractorRow(symptom) {
+  return {
+    ctcae_term: symptom.ctcaeTerm,
+    spanish_label: symptom.spanishLabel,
+    patient_phrases_validated: symptom.patientPhrasesValidated,
+    observed_fields_schema: symptom.observedFieldsSchema,
+    critical_field: symptom.criticalField,
+    grades: symptom.grades,
+  };
+}
+
+async function loadExtractorConfig() {
+  const cached = extractorConfigCache.get(CONFIG_CACHE_KEY);
+  if (cached) return cached;
+
+  const records = await getRecordByFilter(Config_ID, CONFIG_TABLE, 'TRUE()');
+  if (!records) throw new Error('extractor config could not be read from the Config base');
+  const config = Object.fromEntries(records.map((r) => [r.config, r.value]));
+  extractorConfigCache.set(CONFIG_CACHE_KEY, config);
+  return config;
+}
+
+async function getExtractorInputTemplate() {
+  const config = await loadExtractorConfig();
+  return {
+    default: config.EXTRACTOR_INPUT_TEMPLATE_DEFAULT || '',
+    symptom: config.EXTRACTOR_INPUT_TEMPLATE_SYMPTOM || '',
+  };
+}
+
+async function getConversationalTokens() {
+  const config = await loadExtractorConfig();
+  return safeParseArray(config.CONVERSATIONAL_TOKENS) || [];
+}
+
+module.exports = {
+  extractClinicalMentions,
+  getConversationalTokens,
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peopl-health/nexus",
-  "version": "6.0.0-dev.619",
+  "version": "6.0.0-dev.634",
   "description": "Core messaging and assistant library for WhatsApp communication platforms",
   "keywords": [
     "whatsapp",