bps-mcp-server 0.16.3 → 0.17.0

package/dist/tools/smart.tools.js

@@ -2,8 +2,12 @@ import { z } from "zod/v4";
 import { formatDynamicData } from "../services/data-formatter.js";
 import { appendAttribution } from "../services/attribution.js";
 import { logger } from "../utils/logger.js";
-import { lookupVar, learnVar, invalidateVar, lookupPeriod, learnPeriod, invalidatePeriod, normalizeKeyword, } from "../services/learning.js";
+import { htmlTableToMarkdown } from "../utils/html.js";
+import { lookupVar, learnVar, invalidateVar, learnPeriod, invalidatePeriod, normalizeKeyword, } from "../services/learning.js";
 import { detectIntent, generateResultHints } from "../services/intent-detector.js";
+import { computeRelevanceScore, expandQueryTerms, resolveVariableForYears, searchVariables, } from "../services/variable-search.js";
+import { filterRelevantVervar } from "../services/dynamic-data-utils.js";
+import { formatUnit, getPeriodId, getPeriodLabel, } from "../client/fields.js";
 /**
  * AI-friendly shortcut tools that reduce multi-step workflows to single calls.
  */
@@ -11,6 +15,9 @@ export function registerSmartTools(server, client, resolver, config, store) {
     // ---------- find_variable ----------
     server.tool("find_variable", `Cari variabel data BPS berdasarkan kata kunci. Mencari di semua subjek yang relevan.
 
+Bukan untuk ranking: bila user minta "10 kabupaten/kota ... tertinggi", panggil
+\`get_ranking(level="kabkota")\` — tool itu sudah tahu var_id tiap provinsi.
+
 Contoh penggunaan:
 - find_variable(keyword="penduduk") → variabel terkait jumlah penduduk
 - find_variable(keyword="kemiskinan", domain="3500") → variabel kemiskinan di Jawa Timur
@@ -23,31 +30,24 @@ Setelah mendapat var_id dari tool ini, gunakan get_dynamic_data untuk mengambil
     }, async ({ keyword, domain, subject }) => {
         try {
             const allVars = [];
-            const kw = keyword.toLowerCase();
             if (subject) {
-                // Search within specific subject
-                await searchVariablesInSubject(client, domain, subject, kw, allVars);
+                await searchVariablesInSubject(client, domain, subject, normalizeKeyword(keyword), allVars);
             }
             else {
-                // Use keyword → subject mapping + subject title matching
-                const mappedIds = getSubjectIdsForKeyword(kw);
-                const subjects = await client.listSubjects(domain);
-                const matchedIds = subjects.data
-                    .filter(s => {
-                    const t = s.title.toLowerCase();
-                    return kw.split(/\s+/).some(w => w.length > 2 && t.includes(w)) || t.includes(kw);
-                })
-                    .map(s => s.sub_id);
-                const subjectIds = [...new Set([...mappedIds, ...matchedIds])];
-                // Search in relevant subjects first
-                for (const subId of subjectIds.slice(0, 5)) {
-                    await searchVariablesInSubject(client, domain, subId, kw, allVars);
-                    if (allVars.length >= 15)
-                        break;
-                }
-                // If no results from relevant subjects, try without subject filter
-                if (allVars.length === 0) {
-                    await searchVariablesInSubject(client, domain, undefined, kw, allVars);
+                const candidates = await searchVariables(client, domain, keyword, {
+                    maxSubjects: 8,
+                    maxPagesPerSubject: 6,
+                    maxCandidates: 20,
+                });
+                for (const candidate of candidates) {
+                    if (allVars.some((item) => item.var_id === candidate.var_id))
+                        continue;
+                    allVars.push({
+                        var_id: candidate.var_id,
+                        title: candidate.title,
+                        sub_name: candidate.sub_name,
+                        unit: candidate.unit,
+                    });
                 }
             }
             if (allVars.length === 0) {
@@ -65,7 +65,8 @@ Setelah mendapat var_id dari tool ini, gunakan get_dynamic_data untuk mengambil
                 "",
             ];
             for (const v of allVars.slice(0, 15)) {
-                lines.push(`- **${v.title}** (var_id: \`${v.var_id}\`) — Subjek: ${v.sub_name}${v.unit ? ` — Satuan: ${v.unit}` : ""}`);
+                const unit = formatUnit(v.unit);
+                lines.push(`- **${v.title}** (var_id: \`${v.var_id}\`) — Subjek: ${v.sub_name}${unit ? ` — Satuan: ${unit}` : ""}`);
                 if (v.def)
                     lines.push(`  _${v.def.substring(0, 150)}_`);
             }
@@ -82,6 +83,12 @@ Setelah mendapat var_id dari tool ini, gunakan get_dynamic_data untuk mengambil
     server.tool("find_data", `Tool utama untuk AI: cari dan ambil data BPS dalam satu langkah.
 Secara otomatis: detect intent → resolve wilayah → cari variabel → ambil data.
 
+⚠️ JANGAN pakai tool ini untuk PERINGKAT/top-N wilayah ("10 kabupaten/kota
+tertinggi", "5 provinsi termiskin", "daerah dengan ... terbesar"). Panggil
+\`get_ranking\` langsung — untuk kabupaten/kota pakai level="kabkota". Menyapu
+sendiri daftar provinsi + variabel akan memakan 100+ panggilan dan tetap
+kehilangan sebagian wilayah. find_data hanya untuk SATU wilayah.
+
 ## Intent Detection (otomatis):
 - **Single value** (angka spesifik) → find_data
 - **Comparison** ("bandingkan X dan Y") → delegate ke compare_data
@@ -97,12 +104,11 @@ Secara otomatis: detect intent → resolve wilayah → cari variabel → ambil d
 | Kemiskinan (jumlah) | find_data atau get_dynamic_data | 183, 185 |
 | Kemiskinan (%) | find_data atau get_dynamic_data | 184, 192 |
 | Pengangguran (TPT %) | get_dynamic_data(var="543") | 543 |
-| Pengangguran (jumlah) | get_dynamic_data(var="674") | 674 |
-| Inflasi (YoY) | list_strategic_indicators | - |
+| Inflasi (YoY) | list_strategic_indicators atau get_dynamic_data | 2263 |
 | Pertumbuhan ekonomi | list_strategic_indicators | - |
-| IPM | list_strategic_indicators atau get_dynamic_data | 1706 |
+| IPM | list_strategic_indicators atau get_dynamic_data | 413 |
 | Gini Rasio | get_dynamic_data(var="98") | 98 |
-| Jumlah Penduduk | get_dynamic_data(var="1452") | 1452 |
+| Jumlah Penduduk | get_dynamic_data(var="2781") | 2781 |
 | PDRB | list_strategic_indicators(domain=kode_prov) | - |
 | Ekspor/Impor | list_strategic_indicators atau get_trade_data | - |
 | Agama/Religi | find_data atau list_static_tables | - |
@@ -129,9 +135,13 @@ Contoh:
             const intent = detectIntent(query, region, year);
             logger.debug(`find_data: detected intent="${intent.intent}" confidence=${intent.confidence.toFixed(2)} tool="${intent.suggestedTool}"`);
             // Prompt Redirection for specialized intents
-            if (intent.intent !== "single_value" && intent.intent !== "table" && intent.intent !== "unknown" && intent.confidence >= 0.6) {
+            const compareRegion1 = intent.extractedParams.region1;
+            const compareRegion2 = intent.extractedParams.region2;
+            const canRedirect = intent.intent !== "comparison" || (!!compareRegion1 && !!compareRegion2);
+            if (canRedirect && intent.intent !== "single_value" && intent.intent !== "table" && intent.intent !== "unknown" && intent.confidence >= 0.6) {
                 let redirectionText = "";
                 if (intent.intent === "comparison") {
+                    const regions = `${compareRegion1}, ${compareRegion2}`;
                     redirectionText = `**Menyalin intensi pencarian: Perbandingan Wilayah**\n` +
                         `Query Anda tampaknya merupakan perbandingan data antar wilayah. Silakan panggil tool \`compare_data\` untuk hasil terbaik.\n\n` +
                         `**Rekomendasi pemanggilan tool:**\n` +
@@ -140,7 +150,7 @@ Contoh:
                         `  "name": "compare_data",\n` +
                         `  "arguments": {\n` +
                         `    "query": "${query}",\n` +
-                        `    "regions": "${region}"\n` +
+                        `    "regions": "${regions}"\n` +
                         (year ? `    ,"year": "${year}"\n` : "") +
                         `  }\n` +
                         `}\n` +
@@ -161,6 +171,7 @@ Contoh:
                         `\`\`\``;
                 }
                 else if (intent.intent === "ranking") {
+                    const kabkotaLevel = /\b(kabupaten|kabkota|kab\/kota|kabupaten\/kota)\b/i.test(query);
                     redirectionText = `**Menyalin intensi pencarian: Peringkat/Ranking**\n` +
                         `Query Anda tampaknya meminta peringkat data wilayah. Silakan panggil tool \`get_ranking\` untuk hasil terbaik.\n\n` +
                         `**Rekomendasi pemanggilan tool:**\n` +
@@ -170,8 +181,8 @@ Contoh:
                         `  "arguments": {\n` +
                         `    "query": "${query}",\n` +
                         `    "top_n": 10,\n` +
-                        `    "order": "highest"\n` +
-                        `  }\n` +
+                        `    "order": "highest",\n    "level": "${kabkotaLevel ? "kabkota" : "provinsi"}"` +
+                        `\n  }\n` +
                         `}\n` +
                         `\`\`\``;
                 }
@@ -217,98 +228,52 @@ Contoh:
                     };
                 }
             }
-            // Step 2: Find variable via 3-layer lookup
+            // Headline topics without a year are best answered by strategic indicators
+            // (e.g. "inflasi" → the latest national YoY figure, not 150 kabupaten rows).
             const kw = normalizeKeyword(query);
-            // Skip cached lookup if query explicitly asks for breakdown (kab/kota)
-            // because cached var is likely the aggregate, not the breakdown
-            const asksBreakdown = /\b(kabupaten|kab[/\s]kota|per\s*kab)\b/.test(kw);
-            let bestVar = asksBreakdown ? null : await lookupVar(query, domain, store);
-            let fromLearning = !!bestVar;
-            const candidates = [];
-            if (!bestVar) {
-                bestVar = await fullSearchVar(client, kw, domain, candidates);
-                fromLearning = false;
+            if (!year && isHeadlineQuery(kw)) {
+                const headline = await tryStrategicIndicators(client, kw, domain, domainName, undefined, true);
+                if (headline)
+                    return headline;
+            }
+            // Step 2: Find variable via learned mapping → relevance search.
+            // When a year is requested, only accept a variable that actually has
+            // that year (otherwise BPS may silently return an older series).
+            const asksBreakdown = /\b(kabupaten|kebupaten|kab[/\s]kota|kecamatan|per\s*kab|per\s*kec|desa|kelurahan)\b/.test(query.toLowerCase());
+            const yearList = year
+                ? year.split(/[,;]/).map((value) => value.trim()).filter(Boolean)
+                : [];
+            const learned = asksBreakdown ? null : await lookupVar(query, domain, store);
+            const resolved = await resolveVariableForYears(client, store, query, domain, yearList, {
+                skipLearned: asksBreakdown,
+                maxCandidates: 20,
+            });
+            let bestVar = resolved?.varData ?? null;
+            let resolvedPeriods = resolved?.periods ?? [];
+            const candidates = resolved?.candidates ?? [];
+            let fromLearning = !!learned && !!bestVar && learned.var_id === bestVar.var_id;
+            const matchedYears = resolved?.matchedYears ?? [];
+            const missingYears = yearList.filter((value) => !matchedYears.includes(value));
+            if (yearList.length > 0 && matchedYears.length === 0) {
+                bestVar = null;
+                resolvedPeriods = [];
             }
             if (!bestVar) {
                 // Fallback: try strategic indicators
                 const indResult = await tryStrategicIndicators(client, kw, domain, domainName, year);
                 if (indResult)
                     return indResult;
-                // Fallback: try static tables with multiple keyword strategies
+                // Fallback: try static tables with keyword strategies
                 try {
-                    // Strategy 1: Try with normalized keyword
-                    let tables = await safeListStaticTables(client, domain, kw);
-                    // Strategy 2: If empty and keyword has multiple words, try splitting or using broader parts
-                    if (tables.length === 0 && kw.split(/\s+/).length > 1) {
-                        const words = kw.split(/\s+/);
-                        // Try the last 2 words (e.g. "sensus ekonomi")
-                        const lastTwo = words.slice(-2).join(" ");
-                        tables = await safeListStaticTables(client, domain, lastTwo);
-                        if (tables.length === 0) {
-                            // Try the first 2 words (e.g. "jumlah usaha")
-                            const firstTwo = words.slice(0, 2).join(" ");
-                            tables = await safeListStaticTables(client, domain, firstTwo);
-                        }
-                    }
-                    // Strategy 3: If empty and keyword is short, try broader keyword "penduduk"
-                    if (tables.length === 0 && kw.split(/\s+/).length <= 2 && kw !== "penduduk") {
-                        tables = await safeListStaticTables(client, domain, "penduduk");
-                    }
-                    // Strategy 4: If still empty, try without keyword filter
-                    if (tables.length === 0) {
-                        tables = await safeListStaticTables(client, domain, undefined);
-                    }
-                    // Strategy 5: If kab/kota and still empty, try parent province
-                    if (tables.length === 0 && domain.length === 4 && !domain.endsWith("00")) {
-                        const parentDomain = domain.slice(0, 2) + "00";
-                        tables = await safeListStaticTables(client, parentDomain, kw);
-                        if (tables.length === 0 && kw.split(/\s+/).length > 1) {
-                            const words = kw.split(/\s+/);
-                            const lastTwo = words.slice(-2).join(" ");
-                            tables = await safeListStaticTables(client, parentDomain, lastTwo);
-                        }
-                        if (tables.length === 0) {
-                            tables = await safeListStaticTables(client, parentDomain, "penduduk");
-                        }
-                    }
-                    if (tables.length > 0) {
-                        // Find best matching table
-                        const searchTerms = [kw, "penduduk", ...kw.split(/\s+/).filter(w => w.length > 2)];
-                        const bestTable = tables.find(t => {
-                            const titleLower = t.title.toLowerCase();
-                            return searchTerms.some(term => titleLower.includes(term));
-                        }) || tables[0];
-                        const tableDetail = await client.getStaticTable(domain, bestTable.table_id);
-                        const tableLines = [
-                            `**Pencarian:** "${query}" di ${domainName}`,
-                            `**Sumber:** Tabel Statis — ${tableDetail.title}`,
-                            "",
-                            tableDetail.table,
-                        ];
-                        if (tableDetail.excel) {
-                            tableLines.push("", `**Download Excel:** ${tableDetail.excel}`);
-                        }
-                        return { content: [{ type: "text", text: appendAttribution(tableLines.join("\n")) }] };
+                    const staticText = await staticTableFallback(client, domain, domainName, kw, query);
+                    if (staticText) {
+                        return { content: [{ type: "text", text: appendAttribution(staticText) }] };
                     }
-                    // Static tables empty — suggest AllStats search with optimized query
-                    const allStatsQuery = `jumlah penduduk menurut agama ${domainName}`;
-                    return {
-                        content: [{
-                                type: "text",
-                                text: appendAttribution(`**Pencarian:** "${query}" di ${domainName}\n\n` +
-                                    `Data untuk topik ini tidak tersedia sebagai dynamic data atau static table di BPS WebAPI.\n` +
-                                    `Data kemungkinan tersedia di website BPS daerah atau publikasi.\n\n` +
-                                    `**Langkah selanjutnya yang disarankan:**\n` +
-                                    `1. Gunakan \`allstats_search(query="${allStatsQuery}", domain="${domain}", content="table")\` untuk mencari di AllStats Search Engine\n` +
-                                    `2. Gunakan \`allstats_deep_search(query="${allStatsQuery}", domain="${domain}")\` untuk mencari di dalam PDF publikasi\n` +
-                                    `3. Cek langsung website BPS daerah (format URL bervariasi per wilayah)\n\n` +
-                                    `**Catatan:** Beberapa data (seperti agama) hanya tersedia di website BPS daerah, tidak di WebAPI.`),
-                            }],
-                    };
                 }
                 catch (staticTableError) {
                     logger.debug(`find_data: static table fallback failed: ${staticTableError instanceof Error ? staticTableError.message : "unknown"}`);
                 }
+                const allStatsQuery = `${query} ${domainName}`.trim();
                 return {
                     content: [{
                             type: "text",
@@ -316,16 +281,19 @@ Contoh:
                                 `**Saran:**\n` +
                                 `1. Gunakan \`find_variable(keyword="${query}", domain="${domain}")\` untuk mencari variabel yang lebih spesifik\n` +
                                 `2. Gunakan \`search(keyword="${query}")\` untuk pencarian lebih luas\n` +
-                                `3. Gunakan \`list_strategic_indicators(domain="${domain}")\` untuk indikator utama`),
+                                `3. Gunakan \`list_strategic_indicators(domain="${domain}")\` untuk indikator utama\n` +
+                                `4. Gunakan \`allstats_search(query="${allStatsQuery}", content="table")\` untuk mencari tabel di AllStats\n` +
+                                `5. Gunakan \`allstats_deep_search(query="${allStatsQuery}")\` untuk mencari teks di dalam PDF publikasi\n\n` +
+                                `**Catatan:** Beberapa data (seperti agama) hanya tersedia di website BPS daerah, tidak di WebAPI.`),
                         }],
                 };
             }
             // Step 3: Resolve period
             logger.debug(`find_data: using var_id=${bestVar.var_id} (${bestVar.title}) for query="${query}"`);
-            const periodParam = await resolvePeriod(client, store, bestVar.var_id, domain, year);
+            const periodParam = await resolvePeriod(client, store, bestVar.var_id, domain, year, resolvedPeriods);
             // Step 4: Get data
             let result = await client.getDynamicData(domain, String(bestVar.var_id), periodParam);
-            // Self-healing: if data empty and var came from learning, invalidate and retry full search
+            // Self-healing: if data empty and var came from learning, invalidate and retry search
             if ((!result.datacontent || Object.keys(result.datacontent).length === 0) && fromLearning) {
                 logger.debug(`find_data: self-healing — invalidating learned var_id=${bestVar.var_id}`);
                 await invalidateVar(query, domain, store);
@@ -333,91 +301,81 @@ Contoh:
                     for (const y of year.split(","))
                         await invalidatePeriod(bestVar.var_id, domain, y.trim(), store);
                 }
-                // Retry with full search
-                const retryCandidates = [];
-                const retryVar = await fullSearchVar(client, kw, domain, retryCandidates);
+                const retryCandidates = await searchVariables(client, domain, kw, {
+                    maxSubjects: 6,
+                    maxPagesPerSubject: 6,
+                    maxCandidates: 20,
+                });
+                const retryVar = retryCandidates[0];
                 if (retryVar) {
-                    bestVar = retryVar;
+                    bestVar = {
+                        var_id: retryVar.var_id,
+                        title: retryVar.title,
+                        sub_name: retryVar.sub_name,
+                        unit: retryVar.unit,
+                    };
                     fromLearning = false;
                     const retryPeriod = await resolvePeriod(client, store, bestVar.var_id, domain, year);
                     result = await client.getDynamicData(domain, String(bestVar.var_id), retryPeriod);
                     candidates.push(...retryCandidates);
                 }
             }
-            const formatted = formatDynamicData(result, domain, config.defaultLang);
-            const header = `**Pencarian:** "${query}" di ${domainName}${year ? ` (${year})` : ""}\n**Variabel:** ${bestVar.title} (ID: ${bestVar.var_id})\n\n`;
+            // If the best variable has no data, try the next candidates before
+            // falling back to static tables.
+            if ((!result.datacontent || Object.keys(result.datacontent).length === 0) && candidates.length > 1) {
+                for (const candidate of candidates.slice(1, 4)) {
+                    if (candidate.var_id === bestVar.var_id)
+                        continue;
+                    try {
+                        const candidatePeriod = await resolvePeriod(client, store, candidate.var_id, domain, year);
+                        const candidateResult = await client.getDynamicData(domain, String(candidate.var_id), candidatePeriod);
+                        if (candidateResult.datacontent && Object.keys(candidateResult.datacontent).length > 0) {
+                            bestVar = {
+                                var_id: candidate.var_id,
+                                title: candidate.title,
+                                sub_name: candidate.sub_name,
+                                unit: candidate.unit,
+                            };
+                            result = candidateResult;
+                            fromLearning = false;
+                            logger.debug(`find_data: candidate fallback → var_id=${bestVar.var_id}`);
+                            break;
+                        }
+                    }
+                    catch (candidateError) {
+                        logger.debug(`find_data: candidate var_id=${candidate.var_id} failed: ${candidateError instanceof Error ? candidateError.message : "unknown"}`);
+                    }
+                }
+            }
             // Fallback for kab/kota domains: try parent province if no data
             if ((!result.datacontent || Object.keys(result.datacontent).length === 0) &&
                 domain !== "0000" && domain.length === 4 && !domain.endsWith("00")) {
                 const parentDomain = domain.slice(0, 2) + "00";
-                const parentCandidates = [];
-                const parentVar = await lookupVar(query, parentDomain, store) || await fullSearchVar(client, kw, parentDomain, parentCandidates);
+                const parentCandidates = await searchVariables(client, parentDomain, kw, {
+                    maxSubjects: 6,
+                    maxPagesPerSubject: 6,
+                    maxCandidates: 10,
+                });
+                const parentVar = (await lookupVar(query, parentDomain, store)) || parentCandidates[0] || null;
                 if (parentVar) {
                     const parentPeriod = await resolvePeriod(client, store, parentVar.var_id, parentDomain, year);
                     const parentResult = await client.getDynamicData(parentDomain, String(parentVar.var_id), parentPeriod);
                     if (parentResult.datacontent && Object.keys(parentResult.datacontent).length > 0) {
-                        const parentFormatted = formatDynamicData(parentResult, parentDomain, config.defaultLang);
+                        const relevantParent = filterRelevantVervar(parentResult, parentDomain);
+                        const parentFormatted = formatDynamicData(relevantParent, parentDomain, config.defaultLang, {
+                            includeAttribution: false,
+                        });
                         const parentHeader = `**Pencarian:** "${query}" di ${domainName}${year ? ` (${year})` : ""}\n**Variabel:** ${parentVar.title} (ID: ${parentVar.var_id})\n_Data diambil dari domain provinsi induk (${parentDomain})_\n\n`;
-                        return { content: [{ type: "text", text: parentHeader + parentFormatted }] };
+                        return { content: [{ type: "text", text: appendAttribution(parentHeader + parentFormatted) }] };
                     }
                 }
             }
             // Still no data after retry — try static tables as fallback
             if (!result.datacontent || Object.keys(result.datacontent).length === 0) {
-                // Try static tables fallback with multiple strategies
                 try {
-                    // Strategy 1: Try with normalized keyword
-                    let tables = await safeListStaticTables(client, domain, kw);
-                    // Strategy 2: If empty and keyword has multiple words, try splitting or using broader parts
-                    if (tables.length === 0 && kw.split(/\s+/).length > 1) {
-                        const words = kw.split(/\s+/);
-                        // Try the last 2 words (e.g. "sensus ekonomi")
-                        const lastTwo = words.slice(-2).join(" ");
-                        tables = await safeListStaticTables(client, domain, lastTwo);
-                        if (tables.length === 0) {
-                            // Try the first 2 words (e.g. "jumlah usaha")
-                            const firstTwo = words.slice(0, 2).join(" ");
-                            tables = await safeListStaticTables(client, domain, firstTwo);
-                        }
-                    }
-                    // Strategy 3: If empty and keyword is short, try broader keyword "penduduk"
-                    if (tables.length === 0 && kw.split(/\s+/).length <= 2 && kw !== "penduduk") {
-                        tables = await safeListStaticTables(client, domain, "penduduk");
-                    }
-                    // Strategy 4: If still empty, try without keyword
-                    if (tables.length === 0) {
-                        tables = await safeListStaticTables(client, domain, undefined);
-                    }
-                    // Strategy 5: If kab/kota, try parent province
-                    if (tables.length === 0 && domain.length === 4 && !domain.endsWith("00")) {
-                        const parentDomain = domain.slice(0, 2) + "00";
-                        tables = await safeListStaticTables(client, parentDomain, kw);
-                        if (tables.length === 0 && kw.split(/\s+/).length > 1) {
-                            const words = kw.split(/\s+/);
-                            const lastTwo = words.slice(-2).join(" ");
-                            tables = await safeListStaticTables(client, parentDomain, lastTwo);
-                        }
-                        if (tables.length === 0) {
-                            tables = await safeListStaticTables(client, parentDomain, "penduduk");
-                        }
-                    }
-                    if (tables.length > 0) {
-                        const searchTerms = [kw, "penduduk", ...kw.split(/\s+/).filter(w => w.length > 2)];
-                        const bestTable = tables.find(t => {
-                            const titleLower = t.title.toLowerCase();
-                            return searchTerms.some(term => titleLower.includes(term));
-                        }) || tables[0];
-                        const tableDetail = await client.getStaticTable(domain, bestTable.table_id);
-                        const tableLines = [
-                            `**Pencarian:** "${query}" di ${domainName}`,
-                            `**Sumber:** Tabel Statis — ${tableDetail.title}`,
-                            "",
-                            tableDetail.table,
-                        ];
-                        if (tableDetail.excel) {
-                            tableLines.push("", `**Download Excel:** ${tableDetail.excel}`);
-                        }
-                        return { content: [{ type: "text", text: appendAttribution(tableLines.join("\n")) }] };
+                    const staticText = await staticTableFallback(client, domain, domainName, kw, query);
+                    if (staticText) {
+                        return { content: [{ type: "text", text: appendAttribution(staticText) }] };
                     }
                 }
                 catch (staticTableError) {
@@ -427,22 +385,35 @@ Contoh:
                     `Data untuk variabel "${bestVar.title}" tidak tersedia${year ? ` untuk tahun ${year}` : ""} di ${domainName}.`,
                     "",
                 ];
-                if (candidates.length > 1) {
+                const alternatives = candidates.filter((candidate) => candidate.var_id !== bestVar.var_id);
+                if (alternatives.length > 0) {
                     altLines.push("**Variabel alternatif yang ditemukan:**");
-                    for (const c of candidates.slice(0, 5)) {
-                        altLines.push(`- ${c.title} (var_id: \`${c.var_id}\`)${c.unit ? ` — ${c.unit}` : ""}`);
+                    for (const c of alternatives.slice(0, 5)) {
+                        const unit = formatUnit(c.unit);
+                        altLines.push(`- ${c.title} (var_id: \`${c.var_id}\`)${unit ? ` — ${unit}` : ""}`);
                     }
                     altLines.push("");
                     altLines.push("Gunakan `get_dynamic_data` dengan var_id di atas untuk mencoba variabel lain.");
                 }
                 return { content: [{ type: "text", text: appendAttribution(altLines.join("\n")) }] };
             }
-            // Success — learn the variable mapping
+            // Success — learn the variable mapping and show the aggregate for large
+            // region breakdowns so the output stays useful for an LLM.
             await learnVar(query, domain, bestVar, store);
-            // Generate result hints
+            const relevantResult = filterRelevantVervar(result, domain, domainName, asksBreakdown ? 10_000 : 10);
+            const filteredNote = relevantResult === result
+                ? ""
+                : "\n_Menampilkan agregat wilayah. Gunakan get_dynamic_data untuk rincian lengkap._\n";
+            const formatted = formatDynamicData(relevantResult, domain, config.defaultLang, {
+                includeAttribution: false,
+            });
+            const missingNote = missingYears.length > 0 && matchedYears.length > 0
+                ? `\n_Catatan: tahun ${missingYears.join(", ")} tidak tersedia untuk variabel ini; data yang ditampilkan hanya tahun ${matchedYears.join(", ")}._\n`
+                : "";
+            const header = `**Pencarian:** "${query}" di ${domainName}${year ? ` (${year})` : ""}\n**Variabel:** ${bestVar.title} (ID: ${bestVar.var_id})\n${filteredNote}${missingNote}\n`;
             const hints = generateResultHints(query, domain, domainName, bestVar.var_id, bestVar.title);
             const hintsText = hints.length > 0 ? "\n\n**💡 Tips Lanjutan:**\n" + hints.join("\n") : "";
-            return { content: [{ type: "text", text: header + formatted + hintsText }] };
+            return { content: [{ type: "text", text: appendAttribution(header + formatted + hintsText) }] };
         }
         catch (error) {
             const msg = error instanceof Error ? error.message : "Gagal mengambil data";
@@ -450,296 +421,273 @@ Contoh:
         }
     });
 }
-// Common keyword → subject ID mapping (BPS subject IDs are stable across domains)
-const KEYWORD_SUBJECTS = {
-    pengangguran: [6], tenaga: [6], kerja: [6], tpak: [6], angkatan: [6],
-    miskin: [23], kemiskinan: [23], gini: [23], ketimpangan: [23],
-    penduduk: [12], kependudukan: [12], fertilitas: [12], migrasi: [12],
-    inflasi: [3], harga: [3], ihk: [3],
-    pdrb: [52], ekonomi: [52, 35], pertumbuhan: [52],
-    ipm: [26], pembangunan: [26],
-    ekspor: [8], impor: [8], perdagangan: [8],
-    pertanian: [55], pangan: [55],
-    industri: [9], manufaktur: [9],
-    kesehatan: [30],
-    pendidikan: [28],
-    pariwisata: [16],
-    agama: [12], religi: [12], keagamaan: [12],
-};
-function getSubjectIdsForKeyword(kw) {
-    const ids = [];
-    for (const [keyword, subIds] of Object.entries(KEYWORD_SUBJECTS)) {
-        if (kw.includes(keyword))
-            ids.push(...subIds);
-    }
-    return [...new Set(ids)];
-}
-/** Full search flow (Layer 3): search subjects → variables → score → return best. */
-async function fullSearchVar(client, kw, domain, candidates) {
-    const mappedSubjectIds = getSubjectIdsForKeyword(kw);
-    const subjects = await client.listSubjects(domain);
-    const relevantSubjects = subjects.data.filter(s => {
-        const titleLower = s.title.toLowerCase();
-        return kw.split(/\s+/).some(w => w.length > 2 && titleLower.includes(w)) || titleLower.includes(kw);
-    });
-    const subjectIdsToSearch = [
-        ...new Set([...mappedSubjectIds, ...relevantSubjects.map(s => s.sub_id)])
-    ];
-    for (const subId of subjectIdsToSearch.slice(0, 5)) {
-        const result = await client.listVariables(domain, subId, undefined, 1, 100);
-        if (!result.data || result.data.length === 0)
-            continue;
-        for (const v of result.data) {
-            const score = computeRelevanceScore(kw, v.title.toLowerCase(), v.sub_name?.toLowerCase() || "");
-            if (score > 0)
-                candidates.push({ var_id: v.var_id, title: v.title, sub_name: v.sub_name, unit: v.unit, score });
-        }
-        // BPS API caps at 10 per page regardless of perpage param — fetch more pages
-        const totalPages = result.page?.pages || 1;
-        for (let page = 2; page <= Math.min(totalPages, 5); page++) {
-            const nextPage = await client.listVariables(domain, subId, undefined, page, 100);
-            if (!nextPage.data || nextPage.data.length === 0)
-                break;
-            for (const v of nextPage.data) {
-                const score = computeRelevanceScore(kw, v.title.toLowerCase(), v.sub_name?.toLowerCase() || "");
-                if (score > 0)
-                    candidates.push({ var_id: v.var_id, title: v.title, sub_name: v.sub_name, unit: v.unit, score });
-            }
-        }
-        if (candidates.length >= 10)
-            break;
-    }
-    if (candidates.length === 0) {
-        const domainsToSearch = domain === "0000" ? ["0000"] : [domain, "0000"];
-        for (const searchDomain of domainsToSearch) {
-            for (let page = 1; page <= 2; page++) {
-                const result = await client.listVariables(searchDomain, undefined, undefined, page, 100);
-                if (!result.data || result.data.length === 0)
-                    break;
-                for (const v of result.data) {
-                    const score = computeRelevanceScore(kw, v.title.toLowerCase(), v.sub_name?.toLowerCase() || "");
-                    if (score > 0)
-                        candidates.push({ var_id: v.var_id, title: v.title, sub_name: v.sub_name, unit: v.unit, score });
-                }
-                if (candidates.length >= 5)
-                    break;
-                if (result.data.length < 100)
-                    break;
-            }
-            if (candidates.length >= 5)
-                break;
-        }
-    }
-    candidates.sort((a, b) => b.score - a.score);
-    if (candidates[0]) {
-        return { var_id: candidates[0].var_id, title: candidates[0].title, sub_name: candidates[0].sub_name, unit: candidates[0].unit };
-    }
-    return null;
+/** Headline topics that are best answered by strategic indicators when no year is given. */
+const HEADLINE_PATTERNS = [
+    /\binflasi\b/,
+    /\bpertumbuhan ekonomi\b/,
+    /\bpengangguran\b/,
+    /\bkemiskinan\b/,
+    /\bmiskin\b/,
+    /\bgini\b/,
+    /\bketimpangan\b/,
+    /\bipm\b/,
+    /\bindeks pembangunan manusia\b/,
+    /\bekspor\b/,
+    /\bimpor\b/,
+    /\bneraca perdagangan\b/,
+];
+function isHeadlineQuery(normalizedQuery) {
+    return HEADLINE_PATTERNS.some((pattern) => pattern.test(normalizedQuery));
 }
 /** Resolve year to period IDs, using learning store first. */
-async function resolvePeriod(client, store, varId, domain, year) {
+async function resolvePeriod(client, store, varId, domain, year, knownPeriods) {
     if (!year) {
         // No year specified — get latest available period
         try {
-            const periods = await client.listPeriods(domain, varId);
+            const periods = knownPeriods && knownPeriods.length > 0
+                ? knownPeriods
+                : await client.listPeriods(domain, varId, 1);
             if (periods.length > 0) {
                 // periods are typically sorted descending; take the first (latest)
-                const latest = periods[0];
-                return String(latest.th_id);
+                return getPeriodId(periods[0]) || undefined;
             }
         }
         catch { /* fall through */ }
         return undefined;
     }
-    const yearNums = year.split(",").map(y => y.trim());
-    const learnedPeriods = [];
-    for (const y of yearNums) {
-        const learned = await lookupPeriod(varId, domain, y, store);
-        if (learned)
-            learnedPeriods.push(learned);
-    }
-    if (learnedPeriods.length === yearNums.length) {
-        return learnedPeriods.join(",");
-    }
-    // Fallback: call list_periods API
+    const yearNums = year.split(/[,;]/).map(y => y.trim()).filter(Boolean);
     try {
-        const periods = await client.listPeriods(domain, varId);
+        const periods = knownPeriods && knownPeriods.length > 0
+            ? knownPeriods
+            : await listAllPeriodsSafe(client, domain, varId);
         if (periods.length > 0) {
             const matchingPeriods = periods.filter(p => {
-                const pAny = p;
-                const thName = String(pAny.th_name || pAny.th || "");
-                const thVal = String(pAny.val || "");
-                const thId = String(p.th_id);
-                return yearNums.some(y => thName.includes(y) || thVal.includes(y) || thId === y);
+                const label = getPeriodLabel(p);
+                return yearNums.some(y => label.includes(y));
             });
             if (matchingPeriods.length > 0) {
-                // Learn period mappings
                 for (const p of matchingPeriods) {
-                    const pAny = p;
-                    const thName = String(pAny.th_name || pAny.th || pAny.val || "");
-                    const matchedYear = yearNums.find(y => thName.includes(y));
-                    if (matchedYear) {
-                        await learnPeriod(varId, domain, matchedYear, String(p.th_id), store);
+                    const label = getPeriodLabel(p);
+                    const id = getPeriodId(p);
+                    const matchedYear = yearNums.find(y => label.includes(y));
+                    if (matchedYear && id) {
+                        await learnPeriod(varId, domain, matchedYear, id, store);
                     }
                 }
-                return matchingPeriods.map(p => String(p.th_id)).join(",");
+                return matchingPeriods.map(p => getPeriodId(p)).filter(Boolean).join(",");
             }
         }
     }
     catch {
-        // If period lookup fails, use raw year value
+        // If period lookup fails, no usable period.
+    }
+    return undefined;
+}
+async function listAllPeriodsSafe(client, domain, varId) {
+    const maybeAll = client.listAllPeriods;
+    if (typeof maybeAll === "function") {
+        return maybeAll.call(client, domain, varId);
     }
-    return year;
+    return client.listPeriods(domain, varId);
 }
-/** Try strategic indicators as fallback. */
-async function tryStrategicIndicators(client, kw, domain, domainName, year) {
+/** Try strategic indicators as fallback (also used as a headline shortcut). */
+async function tryStrategicIndicators(client, kw, domain, domainName, year, headlineNote = false) {
     const indicators = await client.listStrategicIndicators(domain);
     if (!indicators.data || indicators.data.length === 0)
         return null;
-    for (const ind of indicators.data) {
-        const titleLower = ind.title.toLowerCase();
-        if (titleLower.includes(kw) || kw.split(/\s+/).some(w => titleLower.includes(w))) {
-            if (year && !ind.periode.includes(year)) {
-                continue;
-            }
-            const val = typeof ind.value === "number" ? ind.value.toLocaleString("id-ID") : String(ind.value);
-            const lines = [
-                `## ${ind.title}`,
-                `**Wilayah:** ${domainName} (${domain})`,
-                "",
-                `| Periode | Nilai |`,
-                `| --- | --- |`,
-                `| ${ind.periode} | ${val} ${ind.unit} |`,
-            ];
-            if (ind.name) {
-                lines.push("", `_${ind.name}_`);
-            }
-            return { content: [{ type: "text", text: appendAttribution(lines.join("\n")) }] };
+    const terms = expandQueryTerms(kw);
+    let best = null;
+    for (const indicator of indicators.data) {
+        const titleLower = indicator.title.toLowerCase();
+        let score = 0;
+        if (titleLower.includes(kw))
+            score += 40;
+        for (const term of terms) {
+            if (term.length > 2 && titleLower.includes(term))
+                score += 10;
         }
+        if (score === 0)
+            continue;
+        if (year && !indicator.periode.includes(year))
+            continue;
+        if (!best || score > best.score)
+            best = { indicator, score };
     }
-    return null;
+    if (!best)
+        return null;
+    const indicator = best.indicator;
+    const val = typeof indicator.value === "number"
+        ? indicator.value.toLocaleString("id-ID")
+        : String(indicator.value);
+    const unit = formatUnit(indicator.unit);
+    const lines = [
+        `## ${indicator.title}`,
+        `**Wilayah:** ${domainName} (${domain})`,
+        "",
+        `| Periode | Nilai |`,
+        `| --- | --- |`,
+        `| ${indicator.periode} | ${val}${unit ? ` ${unit}` : ""} |`,
+    ];
+    if (indicator.name) {
+        lines.push("", `_${indicator.name}_`);
+    }
+    if (headlineNote) {
+        lines.push("", "_Data headline terbaru. Untuk data historis, gunakan `get_trend` atau `find_data` dengan parameter `year`._");
+    }
+    return { content: [{ type: "text", text: appendAttribution(lines.join("\n")) }] };
 }
 /**
  * Search variables within a subject (or all if undefined) and add matches to results array.
  */
 async function searchVariablesInSubject(client, domain, subject, kw, results) {
-    const kwWords = kw.split(/\s+/).filter(w => w.length > 2);
-    for (let page = 1; page <= 3; page++) {
-        const result = await client.listVariables(domain, subject, undefined, page, 100);
+    let firstPage;
+    try {
+        firstPage = await client.listVariables(domain, subject, undefined, 1, 100);
+    }
+    catch (error) {
+        logger.debug(`searchVariablesInSubject failed: ${error instanceof Error ? error.message : String(error)}`);
+        return;
+    }
+    if (!firstPage.data || firstPage.data.length === 0)
+        return;
+    const pages = Math.min(firstPage.page?.pages ?? 1, 6);
+    const scored = [];
+    for (let page = 1; page <= pages; page++) {
+        const result = page === 1
+            ? firstPage
+            : await client.listVariables(domain, subject, undefined, page, 100);
         if (!result.data || result.data.length === 0)
             break;
-        for (const v of result.data) {
-            const titleLower = v.title.toLowerCase();
-            const defLower = (v.def || "").toLowerCase();
-            // Match if full keyword or any word matches
-            const matches = titleLower.includes(kw) || defLower.includes(kw) ||
-                kwWords.some(w => titleLower.includes(w));
-            if (matches) {
-                // Avoid duplicates
-                if (!results.some(r => r.var_id === v.var_id)) {
-                    results.push({
-                        var_id: v.var_id,
-                        title: v.title,
-                        sub_name: v.sub_name,
-                        unit: v.unit,
-                        def: v.def,
-                    });
-                }
-            }
+        for (const variable of result.data) {
+            const score = computeRelevanceScore(kw, variable.title, variable.sub_name, variable.def);
+            if (score > 0)
+                scored.push({ variable, score });
         }
-        if (results.length >= 15)
-            break;
-        if (result.data.length < 100)
+        if (scored.length >= 30)
             break;
     }
+    scored.sort((a, b) => b.score - a.score);
+    for (const { variable } of scored.slice(0, 15)) {
+        if (results.some((existing) => existing.var_id === variable.var_id))
+            continue;
+        results.push({
+            var_id: variable.var_id,
+            title: variable.title,
+            sub_name: variable.sub_name,
+            unit: variable.unit,
+            def: variable.def,
+        });
+    }
 }
 /**
- * Compute relevance score for a variable title against a search query.
- * Higher score = more relevant.
+ * Search static tables, prefer a title match, and return a compact markdown
+ * table. Returns null when no relevant table is found (rather than showing an
+ * unrelated first result).
  */
-function computeRelevanceScore(query, title, subName) {
-    let score = 0;
-    const queryWords = query.split(/\s+/).filter(w => w.length > 2);
-    // Exact phrase match in title
-    if (title.includes(query))
-        score += 100;
-    // Word-level matches in title
-    let wordMatches = 0;
-    for (const word of queryWords) {
-        if (title.includes(word)) {
-            score += 30;
-            wordMatches++;
+async function staticTableFallback(client, domain, domainName, kw, query) {
+    const words = kw.split(/\s+/).filter(Boolean);
+    let tableDomain = domain;
+    let tables = await safeListStaticTables(client, domain, kw);
+    if (tables.length === 0 && words.length > 1) {
+        const lastTwo = words.slice(-2).join(" ");
+        tables = await safeListStaticTables(client, domain, lastTwo);
+        if (tables.length === 0) {
+            const firstTwo = words.slice(0, 2).join(" ");
+            tables = await safeListStaticTables(client, domain, firstTwo);
         }
-        if (subName.includes(word))
-            score += 15;
     }
-    // Bonus for matching all query words
-    if (queryWords.length > 1 && wordMatches === queryWords.length)
-        score += 40;
-    // Title starts with query
-    if (title.startsWith(query))
-        score += 50;
-    // Prefer "tingkat" or "persentase" variants (main indicators)
-    if (title.includes("tingkat") || title.includes("persentase") || title.includes("jumlah"))
-        score += 20;
-    // Prefer shorter titles (more general/main indicators)
-    if (title.length < 60)
-        score += 15;
-    if (title.length > 100)
-        score -= 20;
-    // Penalize titles with "menurut" (breakdowns are less useful as primary)
-    // UNLESS query explicitly asks for breakdown (kabupaten, kab, kota)
-    const queryAsksBreakdown = query.includes("kabupaten") || query.includes("kab") ||
-        (query.includes("kota") && !query.includes("perkotaan"));
-    const menurutCount = (title.match(/menurut/g) || []).length;
-    if (menurutCount > 1 && !queryAsksBreakdown)
-        score -= 15;
-    // Boost if query asks for kab/kota breakdown and title has it
-    if (queryAsksBreakdown && (title.includes("kabupaten") || title.includes("kab/"))) {
-        score += 60;
+    if (tables.length === 0 && words.length <= 2 && kw !== "penduduk") {
+        tables = await safeListStaticTables(client, domain, "penduduk");
+    }
+    if (tables.length === 0) {
+        tables = await safeListStaticTables(client, domain, undefined);
     }
-    // Prefer "persentase" or "jumlah penduduk miskin" over "garis kemiskinan" or "indeks"
-    if (query.includes("miskin") || query.includes("kemiskinan")) {
-        // If user explicitly asks for "jumlah", prefer jumlah variant
-        if (query.includes("jumlah")) {
-            if (title.includes("jumlah"))
-                score += 40;
-            if (title.includes("persentase"))
-                score -= 10;
+    if (tables.length === 0 && domain.length === 4 && !domain.endsWith("00")) {
+        const parentDomain = `${domain.slice(0, 2)}00`;
+        tableDomain = parentDomain;
+        tables = await safeListStaticTables(client, parentDomain, kw);
+        if (tables.length === 0 && words.length > 1) {
+            tables = await safeListStaticTables(client, parentDomain, words.slice(-2).join(" "));
         }
-        else {
-            if (title.includes("persentase"))
-                score += 40;
-            else if (title.includes("jumlah penduduk miskin"))
-                score += 20;
+        if (tables.length === 0) {
+            tables = await safeListStaticTables(client, parentDomain, "penduduk");
         }
-        if (title.includes("garis kemiskinan") && !query.includes("garis"))
-            score -= 30;
-        if (title.includes("indeks kedalaman") || title.includes("indeks keparahan"))
-            score -= 20;
     }
-    // Prefer "jumlah penduduk menurut agama" for religion queries
-    if (query.includes("agama") || query.includes("religi")) {
-        if (title.includes("jumlah") || title.includes("penduduk"))
+    const asksBreakdown = /\b(kabupaten|kecamatan|desa|kelurahan|per\s*kab|per\s*kec)\b/i.test(query);
+    const searchTerms = [kw, ...words.filter((word) => word.length > 2)];
+    const scoredTables = tables
+        .map((table) => {
+        const title = table.title.toLowerCase();
+        let score = 0;
+        if (!searchTerms.some((term) => title.includes(term)))
+            return { table, score: 0 };
+        score += 20;
+        if (domainName && title.includes(domainName.toLowerCase()))
             score += 30;
-        if (title.includes("menurut agama"))
-            score += 50;
-        if (title.includes("kepercayaan"))
+        if (title.includes("kabupaten"))
             score += 10;
+        if (title.includes("jumlah") || title.includes("penduduk"))
+            score += 5;
+        if (/desa|kelurahan|kecamatan/.test(title)) {
+            score += asksBreakdown ? 15 : -25;
+        }
+        return { table, score };
+    })
+        .filter((entry) => entry.score > 0)
+        .sort((a, b) => b.score - a.score);
+    if (scoredTables.length === 0)
+        return null;
+    let framesetFallback = null;
+    for (const { table } of scoredTables.slice(0, 5)) {
+        let detail;
+        try {
+            detail = await client.getStaticTable(tableDomain, table.table_id);
+        }
+        catch {
+            continue;
+        }
+        const tableText = htmlTableToMarkdown(detail.table);
+        const lines = [
+            `**Pencarian:** "${query}" di ${domainName}`,
+            `**Sumber:** Tabel Statis — ${detail.title}`,
+        ];
+        if (tableDomain !== domain) {
+            lines.push(`_Tabel berasal dari domain provinsi induk (${tableDomain})._`);
+        }
+        lines.push("", tableText);
+        lines.push("", `**Detail/Excel:** get_static_table(domain="${tableDomain}", id=${detail.table_id})`);
+        if (tableText.startsWith("_Tabel ini disajikan BPS sebagai file Excel")) {
+            framesetFallback = framesetFallback ?? lines.join("\n");
+            continue;
+        }
+        return lines.join("\n");
     }
-    return score;
+    return framesetFallback;
 }
 /**
  * Safely lists static tables without throwing on BpsNotFoundError.
  */
-async function safeListStaticTables(client, domain, keyword) {
-    try {
-        const res = await client.listStaticTables(domain, keyword, undefined, undefined, 1);
-        return res.data || [];
-    }
-    catch (err) {
-        logger.debug(`safeListStaticTables failed for keyword="${keyword}": ${err instanceof Error ? err.message : String(err)}`);
-        return [];
+async function safeListStaticTables(client, domain, keyword, maxPages = 3) {
+    const results = [];
+    const seen = new Set();
+    for (let page = 1; page <= maxPages; page++) {
+        try {
+            const res = await client.listStaticTables(domain, keyword, undefined, undefined, page);
+            for (const table of res.data || []) {
+                if (seen.has(table.table_id))
+                    continue;
+                seen.add(table.table_id);
+                results.push(table);
+            }
+            const pages = res.page?.pages ?? 1;
+            if (page >= Math.min(pages, maxPages) || (res.data?.length ?? 0) === 0)
+                break;
+        }
+        catch (err) {
+            logger.debug(`safeListStaticTables failed for keyword="${keyword}" page=${page}: ${err instanceof Error ? err.message : String(err)}`);
+            break;
+        }
     }
+    return results;
 }
 //# sourceMappingURL=smart.tools.js.map