fodda-mcp 1.38.0 → 1.46.97

package/dist/catalogCache.js

@@ -7,6 +7,100 @@
  */
 import axios from 'axios';
 import { cacheClear } from './queryCache.js';
+export function stringifyField(f) {
+    if (!f)
+        return '';
+    if (typeof f === 'string')
+        return f.trim();
+    if (Array.isArray(f)) {
+        return f.map(item => {
+            if (typeof item === 'string')
+                return item;
+            if (item && typeof item === 'object') {
+                return [item.title, item.name, item.description, item.topic].filter(Boolean).join(' ');
+            }
+            return '';
+        }).filter(Boolean).join('. ').trim();
+    }
+    if (typeof f === 'object') {
+        return [f.title, f.name, f.description, f.topic].filter(Boolean).join(' ').trim();
+    }
+    return String(f);
+}
+export function normalizeAnalyst(a) {
+    const analyst_id = String(a.analyst_id || a.id || a.slug || a.name || '').trim();
+    const name = cleanDisplayName(String(a.name || a.analyst_id || a.id || a.slug || '').trim());
+    const description = stringifyField(a.description || a.bio || a.lens || a.standpoint || '');
+    const expert_in = stringifyField(a.expert_in || a.expertIn || a.topic || '');
+    const outside_their_lane = stringifyField(a.outside_their_lane || a.outsideTheirLane || a.blind_spots || a.blindSpots || '');
+    const topics = Array.isArray(a.topics) ? a.topics : (typeof a.topics === 'string' ? a.topics.split(',').map((s) => s.trim()) : undefined);
+    const rawSubType = (a.graphSubType || a.graph_sub_type || a.subType || a.type || a.kind || a.agent_type || '').toString().trim();
+    const cleanId = analyst_id.toLowerCase();
+    // ── 4-Tier Agent Classification ──
+    // 1. C-Suite Agents: corporate executive personas grounded in SEC/earnings
+    const is_c_suite = cleanId === 'brand-cmo' || cleanId === 'brand-ceo' || cleanId === 'brand-cfo' ||
+        /brand-(cmo|ceo|cfo)/i.test(cleanId) || /c-suite|executive/i.test(rawSubType);
+    // 2. Classic Agents: public-domain historical thinkers providing philosophical lenses
+    const is_classic = !is_c_suite && (rawSubType === 'Classic Digital Twin' ||
+        /classic digital twin/i.test(rawSubType) ||
+        a.is_classic_agent === true ||
+        a.category === 'classic_agent');
+    // 3. Human Agents: real living industry figures with consented digital twins
+    const is_human = !is_c_suite && !is_classic && Boolean(rawSubType === 'Digital Twin' ||
+        rawSubType === 'human_agent' ||
+        rawSubType === 'human_twin' ||
+        rawSubType === 'expert_twin' ||
+        a.is_human_agent ||
+        a.is_digital_twin ||
+        a.isVerifiedRealPerson ||
+        a.is_verified_real_person ||
+        a.type === 'human_agent' ||
+        a.type === 'human_twin' ||
+        a.kind === 'human_agent' ||
+        a.kind === 'human_twin' ||
+        a.agent_type === 'human_agent' ||
+        a.agent_type === 'human_twin');
+    let category = 'synthetic_agent';
+    let category_label = 'Synthetic Agent';
+    let twin_type = undefined;
+    if (is_human) {
+        category = 'human_agent';
+        category_label = 'Human Agent';
+        twin_type = 'Expert Digital Twin';
+    }
+    else if (is_classic) {
+        category = 'classic_agent';
+        category_label = 'Classic Agent';
+        twin_type = 'Classic Agent';
+    }
+    else if (is_c_suite) {
+        category = 'c_suite_agent';
+        category_label = 'C-Suite Agent';
+    }
+    const slug = a.expertSlug || a.expert_slug || a.slug || a.analyst_id || (is_human ? a.id : undefined);
+    const consult_tool = is_human ? 'consult_human_agent' : 'consult_analyst';
+    return {
+        ...a,
+        analyst_id,
+        name,
+        description,
+        category,
+        category_label,
+        ...(twin_type ? { twin_type } : {}),
+        type: category,
+        is_human_agent: is_human,
+        is_classic_agent: is_classic,
+        is_c_suite_agent: is_c_suite,
+        is_verified_real_person: is_human,
+        consult_tool,
+        slug: slug ? String(slug).trim() : undefined,
+        ...(expert_in ? { expert_in } : {}),
+        ...(outside_their_lane ? { outside_their_lane } : {}),
+        ...(topics ? { topics } : {}),
+        ...(a.what_they_offer || a.whatTheyOffer || a.askLine ? { what_they_offer: stringifyField(a.what_they_offer || a.whatTheyOffer || a.askLine) } : {}),
+        ...(a.example_questions || a.exampleQuestions || a.exampleQueries ? { example_questions: a.example_questions || a.exampleQuestions || a.exampleQueries } : {})
+    };
+}
 // ---------------------------------------------------------------------------
 // Cache
 // ---------------------------------------------------------------------------
@@ -34,18 +128,103 @@ async function fetchCatalog() {
  * Fetch the analysts from the API. Requires internal auth.
  */
 async function fetchAnalysts() {
-    const url = `${API_BASE_URL}/v1/analysts`;
-    const internalKey = process.env.FODDA_INTERNAL_API_KEY;
+    const url = `${API_BASE_URL}/v1/analysts?include=unclaimed`;
+    const internalKey = process.env.FODDA_INTERNAL_API_KEY || process.env.FODDA_API_KEY;
     const headers = internalKey ? { 'Authorization': `Bearer ${internalKey}` } : {};
     try {
         const response = await axios.get(url, { headers, timeout: 10000 });
-        return Array.isArray(response.data) ? response.data : (response.data.analysts || []);
+        const raw = Array.isArray(response.data) ? response.data : (response.data.analysts || []);
+        return raw.map(normalizeAnalyst);
     }
     catch (err) {
         console.error(`[catalogCache] Failed to fetch analysts: ${err.message}`);
         return [];
     }
 }
+/**
+ * Bridges active individual analysts into the searchable graph catalog.
+ * Solves the "two-store indexing gap" where an analyst exists in the Analysts table
+ * with a backing Neo4j graph, but has no dedicated record in Airtable's Graph List.
+ */
+function bridgeAnalystGraphs(catalog, analysts) {
+    if (!catalog?.graphs)
+        return catalog;
+    const existingIds = new Set(catalog.graphs.map(g => (g.graph_id || '').toLowerCase().trim()));
+    const bridged = [...catalog.graphs];
+    for (const a of analysts) {
+        const id = (a.analyst_id || a.id || '').toLowerCase().trim();
+        const status = (a.status || a.Status || '').toLowerCase().trim();
+        if (!id || (status && status !== 'active'))
+            continue;
+        // Skip synthetic C-suite leads that already span all graphs
+        if (a.is_c_suite_agent || /brand-(cmo|ceo|cfo|analyst)/i.test(id))
+            continue;
+        // Determine target graph IDs for this analyst:
+        // Check backingGraphs first (e.g. 'postpals-expert-graph', 'sic', '2026-macro-trend-graph')
+        const candidateGraphIds = new Set();
+        const backingList = Array.isArray(a.backingGraphs) ? a.backingGraphs : [];
+        for (const rawBg of backingList) {
+            const bg = (rawBg || '').toLowerCase().trim();
+            if (bg && bg !== '*' && !existingIds.has(bg)) {
+                candidateGraphIds.add(bg);
+            }
+        }
+        // If no un-registered backingGraph found, use the analyst ID itself (e.g. 'peter-abraham-bicycles-cycling')
+        if (candidateGraphIds.size === 0 && !existingIds.has(id) && !backingList.includes('*')) {
+            candidateGraphIds.add(id);
+        }
+        if (candidateGraphIds.size === 0)
+            continue;
+        const topics = Array.isArray(a.topics) ? a.topics : [];
+        const expertIn = typeof a.expert_in === 'string' ? a.expert_in : '';
+        const desc = a.description || a.what_they_offer || expertIn || `${a.name} expert knowledge graph`;
+        for (const targetGraphId of candidateGraphIds) {
+            bridged.push({
+                graph_id: targetGraphId,
+                name: a.name,
+                description: desc,
+                curator: a.name,
+                curator_url: '',
+                quality_checker_name: '',
+                quality_checker_title: '',
+                quality_checker_company: '',
+                update_frequency: 'Weekly',
+                domain: expertIn || 'Specialist Intelligence',
+                version: '1.0.0',
+                node_types: ['Trend', 'Article'],
+                relationship_types: ['HAS_EVIDENCE'],
+                graph_type: 'analyst',
+                status: 'live',
+                trend_count: 5,
+                evidence_count: 5,
+                last_synced: new Date().toISOString(),
+                topics: topics.length > 0 ? topics : [targetGraphId],
+                company: a.company || '',
+                headline: a.askLine || expertIn || desc,
+                subhead: expertIn || '',
+                geography: 'Global',
+                icon_url: '',
+                source_url: '',
+                available_as: 'API, MCP',
+                is_playground: false,
+                last_updated: new Date().toISOString(),
+                published_date: null,
+                example_queries: Array.isArray(a.example_questions) ? a.example_questions : [],
+                portrait_url: a.portrait_url || '',
+                core_tension: null,
+                matched_human_twin_slug: id,
+                expert_stance_quote: null,
+                cross_graph_signals: []
+            });
+            existingIds.add(targetGraphId);
+        }
+    }
+    return {
+        ...catalog,
+        graph_count: bridged.length,
+        graphs: bridged
+    };
+}
 /**
  * Initialize the catalog cache. Call once at server startup.
  * Fetches immediately, then sets up hourly refresh.
@@ -53,8 +232,9 @@ async function fetchAnalysts() {
 export async function initCatalogCache() {
     try {
         cacheClear();
-        cachedCatalog = await fetchCatalog();
+        const rawCatalog = await fetchCatalog();
         cachedAnalysts = await fetchAnalysts();
+        cachedCatalog = bridgeAnalystGraphs(rawCatalog, cachedAnalysts);
         lastFetchedAt = Date.now();
         rebuildSearchIndex();
         console.error(`[catalogCache] Loaded ${cachedCatalog.graph_count} graphs from catalog (search index: ${graphSearchTexts.size} entries)`);
@@ -68,8 +248,9 @@ export async function initCatalogCache() {
         clearInterval(refreshInterval);
     refreshInterval = setInterval(async () => {
         try {
-            cachedCatalog = await fetchCatalog();
+            const rawCatalog = await fetchCatalog();
             cachedAnalysts = await fetchAnalysts();
+            cachedCatalog = bridgeAnalystGraphs(rawCatalog, cachedAnalysts);
             lastFetchedAt = Date.now();
             rebuildSearchIndex();
             console.error(`[catalogCache] Refreshed catalog — ${cachedCatalog.graph_count} graphs (search index rebuilt) and ${cachedAnalysts.length} analysts`);
@@ -79,6 +260,13 @@ export async function initCatalogCache() {
         }
     }, CATALOG_REFRESH_MS);
 }
+export function setCachedCatalogForTesting(catalog, analysts) {
+    if (analysts)
+        cachedAnalysts = analysts.map(normalizeAnalyst);
+    cachedCatalog = bridgeAnalystGraphs(catalog, cachedAnalysts);
+    lastFetchedAt = Date.now();
+    rebuildSearchIndex();
+}
 /**
  * Get the cached catalog. Returns null if not yet fetched.
  */
@@ -185,6 +373,18 @@ export function buildDisplayName(g) {
     }
     return g.name;
 }
+/**
+ * Clean display names across MCP surfaces.
+ * Removes retired suffixes such as "(Classic Digital Twin)", "(Digital Twin)", etc.
+ */
+export function cleanDisplayName(name) {
+    if (!name || typeof name !== 'string')
+        return '';
+    return name
+        .replace(/\s*[\(\[]\s*(?:Classic\s+|Expert\s+|Human\s+)?(?:Digital\s+)?Twin\s*[\)\]]/gi, '')
+        .replace(/\s*[\(\[]\s*(?:Classic|Human|Synthetic|C-Suite)\s+Agent\s*[\)\]]/gi, '')
+        .trim();
+}
 /**
  * Build the GRAPH TYPES block from catalog data.
  */
@@ -585,8 +785,30 @@ const QUERY_EXPANSION_MAP = {
     // Pet & Animal Care
     pet: ['pet', 'cpg', 'food', 'wellness', 'retail'],
     pets: ['pet', 'cpg', 'food', 'wellness', 'retail'],
-    dog: ['pet', 'cpg', 'food', 'wellness'],
-    cat: ['pet', 'cpg', 'food', 'wellness'],
+    dog: ['pet', 'cpg', 'food', 'wellness', 'retail'],
+    dogs: ['pet', 'cpg', 'food', 'wellness', 'retail'],
+    cat: ['pet', 'cpg', 'food', 'wellness', 'retail'],
+    cats: ['pet', 'cpg', 'food', 'wellness', 'retail'],
+    petfood: ['pet', 'cpg', 'food', 'wellness', 'retail'],
+    // Marketing, Experiential & Activations
+    activation: ['retail', 'cpg', 'marketing', 'brand', 'experience', 'hospitality', 'culture'],
+    activations: ['retail', 'cpg', 'marketing', 'brand', 'experience', 'hospitality', 'culture'],
+    experiential: ['retail', 'cpg', 'marketing', 'brand', 'experience', 'hospitality', 'culture', 'design'],
+    popup: ['retail', 'cpg', 'fashion', 'experience', 'design'],
+    popups: ['retail', 'cpg', 'fashion', 'experience', 'design'],
+    sampling: ['cpg', 'retail', 'food', 'beverage', 'beauty', 'experience'],
+    sponsorship: ['marketing', 'brand', 'culture', 'retail', 'entertainment'],
+    installation: ['retail', 'design', 'art', 'experience', 'culture'],
+    flagship: ['retail', 'fashion', 'design', 'experience', 'brand'],
+    immersive: ['retail', 'experience', 'design', 'entertainment', 'technology'],
+    stunt: ['marketing', 'brand', 'culture', 'retail'],
+    showcase: ['retail', 'design', 'experience', 'marketing'],
+    marketing: ['retail', 'cpg', 'brand', 'strategy', 'advertising', 'culture'],
+    campaign: ['marketing', 'brand', 'advertising', 'retail', 'cpg'],
+    advertising: ['marketing', 'brand', 'media', 'retail', 'cpg'],
+    promo: ['marketing', 'retail', 'cpg', 'promotion'],
+    promotion: ['marketing', 'retail', 'cpg', 'promotion'],
+    hospitality: ['hospitality', 'hotel', 'dining', 'food', 'experience', 'leisure', 'culture'],
     // Tech, AI & SaaS
     ai: ['technology', 'ai', 'software', 'enterprise', 'innovation'],
     saas: ['technology', 'software', 'enterprise', 'b2b'],
@@ -616,12 +838,15 @@ function scoreClauseRelevance(clause, g) {
     const searchText = graphSearchTexts.get(g.graph_id) || buildSearchableText(g);
     const topicsText = [...(g.topics || []), ...(g.routing_keywords || []), g.name || '', g.graph_id || ''].join(' ').toLowerCase();
     const domainText = (g.domain || '').toLowerCase();
+    const words = new Set(searchText.split(/[^a-z0-9]+/));
+    const topicWords = new Set(topicsText.split(/[^a-z0-9]+/));
+    const domainWords = new Set(domainText.split(/[^a-z0-9]+/));
     let matchedTerms = 0;
     let highValueMatches = 0;
     for (const term of terms) {
-        if (searchText.includes(term)) {
+        if (words.has(term)) {
             matchedTerms++;
-            if (topicsText.includes(term) || domainText.includes(term)) {
+            if (topicWords.has(term) || domainWords.has(term)) {
                 highValueMatches++;
             }
         }
@@ -642,9 +867,9 @@ function scoreClauseRelevance(clause, g) {
         let expMatches = 0;
         let expHighValueMatches = 0;
         for (const exp of expandedSet) {
-            if (searchText.includes(exp)) {
+            if (words.has(exp)) {
                 expMatches++;
-                if (topicsText.includes(exp) || domainText.includes(exp)) {
+                if (topicWords.has(exp) || domainWords.has(exp)) {
                     expHighValueMatches++;
                 }
             }
@@ -679,6 +904,18 @@ function scoreClauseRelevance(clause, g) {
             finalScore = Math.min(finalScore, 0.40); // Cap generic agency floor
         }
     }
+    // Experiential & Activation Tactic Boost
+    const experientialTerms = new Set([
+        'activation', 'activations', 'experiential', 'popup', 'popups', 'sampling',
+        'installation', 'flagship', 'immersive', 'stunt', 'showcase'
+    ]);
+    const queryHasExperiential = terms.some(t => experientialTerms.has(t));
+    if (queryHasExperiential) {
+        const isExperienceGraph = /retail|cpg|brand|marketing|experience|culture|hospitality/i.test(g.graph_id + ' ' + (g.domain || '') + ' ' + (g.topics || []).join(' ') + ' ' + (g.routing_keywords || []).join(' '));
+        if (isExperienceGraph || g.graph_id === 'retail') {
+            finalScore += 0.20; // Experiential Format Boost
+        }
+    }
     // Generalized Narrow-Domain Mismatch Penalty
     const narrowDomains = ['beauty', 'gaming', 'automotive', 'fashion'];
     for (const dom of narrowDomains) {
@@ -716,10 +953,11 @@ export function classifyGraphTier(g) {
     const isRecurring = g.update_frequency &&
         !['One-Off', 'Coming Soon'].includes(g.update_frequency);
     // Domain graphs with recurring updates + experts with recurring updates = Living
-    if ((g.graph_type === 'domain' || g.graph_type === 'expert') && isRecurring) {
+    // ('analyst' = expert graphs typed 'analyst' in the registry, e.g. Digital Twins)
+    if ((g.graph_type === 'domain' || g.graph_type === 'expert' || g.graph_type === 'analyst') && isRecurring) {
         return 'living';
     }
-    if (g.graph_type === 'expert')
+    if (g.graph_type === 'expert' || g.graph_type === 'analyst')
         return 'static_expert';
     if (g.graph_type === 'industry report')
         return 'report';
@@ -740,8 +978,18 @@ export function classifyGraphTier(g) {
  */
 export function getRelevantGraphs(query, minGraphs = 4, maxGraphs = 15, threshold = 0.10) {
     const DEPRECATED_GRAPH_IDS = new Set(['waldo', 'psfk']);
-    const allGraphs = getLiveGraphs().filter(g => (g.graph_type === 'domain' || g.graph_type === 'expert' || g.graph_type === 'industry report')
-        && !DEPRECATED_GRAPH_IDS.has(g.graph_id));
+    // 'analyst' graphs (e.g. Digital Twin expert graphs typed 'analyst' in the
+    // registry) are searchable expert content — excluding them made registry-live
+    // graphs like jeff-longevity invisible to routing (QA 2026-08-20).
+    const ROUTABLE_TYPES = new Set(['domain', 'expert', 'industry report', 'analyst']);
+    const allLive = getLiveGraphs();
+    const allGraphs = allLive.filter(g => ROUTABLE_TYPES.has(g.graph_type) && !DEPRECATED_GRAPH_IDS.has(g.graph_id));
+    // Registry-status visibility: surface live graphs excluded from routing by
+    // type, so an unroutable analyst/expert graph is detectable in logs.
+    const typeExcluded = allLive.filter(g => !ROUTABLE_TYPES.has(g.graph_type) && g.graph_type !== 'supplemental' && g.graph_type !== 'skill');
+    if (typeExcluded.length > 0) {
+        console.error(`[graphRouter] ${typeExcluded.length} live graph(s) excluded by graph_type: ${typeExcluded.map(g => `${g.graph_id}(${g.graph_type})`).join(', ')}`);
+    }
     if (allGraphs.length === 0)
         return [];
     // Runtime check: skip graphs that have no synced content in Neo4j.
@@ -750,13 +998,35 @@ export function getRelevantGraphs(query, minGraphs = 4, maxGraphs = 15, threshol
     // Once the CE pipeline syncs them, they'll automatically start routing.
     const hasContent = (g) => (g.trend_count > 0) || (g.last_synced !== null && g.last_synced !== undefined);
     const syncedGraphs = allGraphs.filter(hasContent);
-    const skippedShells = allGraphs.length - syncedGraphs.length;
+    const shellGraphs = allGraphs.filter(g => !hasContent(g));
+    const skippedShells = shellGraphs.length;
     if (skippedShells > 0) {
-        console.error(`[graphRouter] Skipped ${skippedShells} unsynced graph shell(s) (trend_count=0, last_synced=null)`);
+        console.error(`[graphRouter] Skipped ${skippedShells} unsynced graph shell(s) (trend_count=0, last_synced=null): ${shellGraphs.map(g => g.graph_id).join(', ')}`);
     }
-    // ── Phase 0: Direct name matching ──
+    // ── Phase 0: Direct name matching & phrase routing overrides ──
     const queryLower = query.toLowerCase();
     const directMatchIds = new Set();
+    const isCreatorQuery = /creator economy|content creator|creator monetization|creators report/i.test(queryLower);
+    const isNightlifeQuery = /nightlife|going out|shared experiences|shared social experiences/i.test(queryLower);
+    const eligibleGraphs = syncedGraphs.filter(g => {
+        if (isCreatorQuery && g.graph_id === 'mary-shelley')
+            return false;
+        return true;
+    });
+    if (isCreatorQuery) {
+        for (const g of eligibleGraphs) {
+            if (g.graph_id === 'visa-creators_report-2025' || g.graph_id.includes('youtube') || g.graph_id.includes('tiktok')) {
+                directMatchIds.add(g.graph_id);
+            }
+        }
+    }
+    if (isNightlifeQuery) {
+        for (const g of eligibleGraphs) {
+            if (g.graph_id.includes('bompasparr')) {
+                directMatchIds.add(g.graph_id);
+            }
+        }
+    }
     const matchesWordBoundary = (text, term) => {
         if (!term || term.length < 3)
             return false;
@@ -771,7 +1041,7 @@ export function getRelevantGraphs(query, minGraphs = 4, maxGraphs = 15, threshol
         const rx = new RegExp(`\\b${escaped}\\b`, 'i');
         return rx.test(text);
     };
-    for (const g of syncedGraphs) {
+    for (const g of eligibleGraphs) {
         // Check graph_id (e.g., "alyson-stevens-macro", "sic")
         if (g.graph_id.length > 3 && matchesWordBoundary(queryLower, g.graph_id)) {
             directMatchIds.add(g.graph_id);
@@ -813,7 +1083,7 @@ export function getRelevantGraphs(query, minGraphs = 4, maxGraphs = 15, threshol
         console.error(`[graphRouter] Phase 0 direct name matches: ${[...directMatchIds].join(', ')}`);
     }
     // Score ALL graph types together — domain, expert, industry report compete on merit
-    const scored = syncedGraphs.map(g => {
+    const scored = eligibleGraphs.map(g => {
         const isDirect = directMatchIds.has(g.graph_id);
         return {
             graph: g,
@@ -907,11 +1177,12 @@ const EARNINGS_SECTORS = ['retail', 'technology', 'travel', 'hospitality', 'luxu
 // (economic indicators/FRED, demographic surveys/Census, trade data, food
 // economics + agricultural production, commodity pricing).
 const SUPPLEMENTAL_CATEGORY_KEYWORDS = {
-    macro: ['macro', 'economy', 'economic', 'inflation', 'gdp', 'interest rate', 'consumer confidence', 'consumer spending', 'recession', 'fred', 'cost of living', 'purchasing power', 'unemployment'],
-    demographics: ['demographic', 'demographics', 'population', 'census', 'household income', 'age distribution', 'migration'],
+    macro: ['macro', 'economy', 'economic', 'inflation', 'gdp', 'interest rate', 'consumer confidence', 'consumer sentiment', 'michigan sentiment', 'umcsent', 'consumer spending', 'recession', 'fred', 'cost of living', 'purchasing power', 'unemployment'],
+    demographics: ['demographic', 'demographics', 'population', 'census', 'household income', 'age distribution', 'migration', 'republican', 'democrat', 'bipartisan', 'party affiliation', 'pew', 'npors'],
     trade: ['imports', 'exports', 'trade data', 'trade flows', 'tariff', 'tariffs', 'supply chain'],
     food_economics: ['food prices', 'food costs', 'crop', 'agriculture', 'agricultural', 'nutrition', 'nutritional'],
     commodities: ['commodity', 'commodities', 'raw materials', 'wholesale prices'],
+    science: ['science', 'scientific', 'clinical', 'pubmed', 'openalex', 'clinical trials', 'dermatology', 'peer-reviewed', 'academic', 'study', 'studies', 'journal', 'biomedical'],
 };
 /**
  * Detect an earnings candidate for a query. Deterministic and conservative: