dsh-plugin-show-me-data 0.1.0

package/src/app/overview.js

@@ -0,0 +1,349 @@
+/**
+ * The overview use case: "today's panel" (docs/07 T6.2).
+ *
+ * Responsibilities: resolve the requested range, refresh every visible
+ * indicator (with per-item degradation), apply each indicator's display
+ * transform, compute stats, run the insight rules, rank the noteworthy list, and
+ * return cards that always carry provenance.
+ *
+ * @module app/overview
+ */
+import { applyTransform, stats as computeStats } from '../core/stats/series.js'
+import { buildNoteworthy } from '../core/insight/rank.js'
+import { runRules } from '../core/insight/rules.js'
+import { filterRange, resolveRange } from '../core/time/range.js'
+import { applyDerive } from '../core/stats/derive.js'
+
+/** How many points a card sparkline carries. */
+export const SPARKLINE_POINTS = 40
+
+/**
+ * Expand a set of indicator ids with every operand a derived indicator needs.
+ *
+ * @param {string[]} ids - requested ids.
+ * @param {Record<string, object>} catalogById - catalog index.
+ * @returns {Set<string>} closed id set.
+ */
+export function withOperands(ids, catalogById) {
+  const closed = new Set()
+  const visit = (id) => {
+    if (closed.has(id)) return
+    closed.add(id)
+    for (const operand of catalogById[id]?.derive?.operands ?? []) visit(operand)
+  }
+  for (const id of ids) visit(id)
+  return closed
+}
+
+/**
+ * Build one card's sparkline: the trailing window of transformed values,
+ * normalized to its own min/max so the shape is visible regardless of units.
+ *
+ * @param {Array<{ t: string, v: number }>} points - transformed points.
+ * @param {number} [limit] - point budget.
+ * @returns {number[]} sparkline values.
+ */
+export function sparkline(points, limit = SPARKLINE_POINTS) {
+  if (!Array.isArray(points) || points.length === 0) return []
+  const tail = points.slice(-limit).map((point) => point.v)
+  const min = Math.min(...tail)
+  const max = Math.max(...tail)
+  if (!Number.isFinite(min) || !Number.isFinite(max)) return []
+  if (max === min) return tail.map(() => 0.5)
+  return tail.map((value) => (value - min) / (max - min))
+}
+
+/**
+ * The full display pipeline for one indicator's raw series.
+ *
+ * @param {object} indicator - catalog definition.
+ * @param {object} series - raw (or derived) series.
+ * @returns {{ points: Array<{ t: string, v: number }>, stats: object|undefined }} display points and stats.
+ */
+export function displaySeries(indicator, series) {
+  const transformed = applyTransform(series.points ?? [], indicator.display ?? {})
+  const stats = computeStats(transformed, { decimals: indicator.display?.decimals, freq: indicator.freq })
+  return { points: transformed, stats }
+}
+
+/**
+ * Make a series' reported count describe the points a caller actually gets.
+ *
+ * Statistics stay computed over the whole fetched series — a YoY reading needs
+ * twelve prior points even when the caller asked for three months — but the
+ * count must not. Leaving it series-wide made the digest print
+ * `points=13 missing=1` above a list of 12 dates, and an agent reading the panel
+ * context reported that contradiction before any human noticed it.
+ *
+ * @param {object} indicator - catalog definition.
+ * @param {{ points: Array<{ t: string, v: number }>, stats: object|undefined }} display - display pipeline output.
+ * @param {{ from: string, to: string }} range - requested range.
+ * @returns {object|undefined} stats whose `count` matches the returned points.
+ */
+export function describeReturned(indicator, display, range) {
+  if (display.stats === undefined) return undefined
+  const returned = filterRange(display.points, range)
+  return { ...display.stats, count: returned.length }
+}
+
+/**
+ * Create the overview use case.
+ *
+ * @param {object} deps - dependencies.
+ * @param {object} deps.catalog - indicator catalog.
+ * @param {Record<string, object>} deps.catalogById - id → definition.
+ * @param {object} deps.refresh - refresh service.
+ * @param {import('../ports/clock.js').Clock} deps.clock - clock.
+ * @param {object} deps.runtime - runtime deps passed to adapters ('{ fetch, clock }').
+ * @param {(msg: string, meta?: object) => void} [deps.log] - logger.
+ * @returns {object} overview use case.
+ */
+export function createOverviewUseCase({ catalog, catalogById, refresh, clock, runtime, log = () => {} }) {
+  /**
+   * Resolve the range for a preset.
+   *
+   * @param {string} preset - range preset.
+   * @param {object} [custom] - custom bounds.
+   * @returns {{ preset: string, from: string, to: string }} range.
+   */
+  function rangeFor(preset = '1Y', custom) {
+    return resolveRange(preset, clock.today(), custom)
+  }
+
+  /**
+   * Refresh and assemble every requested indicator.
+   *
+   * @param {{ range?: string, groups?: string[], ids?: string[], limit?: number, custom?: object, force?: boolean }} [request] - overview request.
+   * @returns {Promise<object>} '{ range, metrics, noteworthy, sources, errors, generatedAt }'.
+   */
+  async function overview(request = {}) {
+    const range = rangeFor(request.range ?? '1Y', request.custom)
+    const groups = request.groups
+    // Selecting a derived indicator implicitly selects its operands: asking for
+    // `us.real10y` must fetch the two rates it is built from.
+    const wanted = request.ids === undefined ? undefined : withOperands(request.ids, catalogById)
+    const selected = catalog.filter((indicator) => {
+      if (wanted !== undefined) return wanted.has(indicator.id)
+      if (groups !== undefined && !groups.includes(indicator.group)) return false
+      return true
+    })
+
+    const direct = selected.filter((indicator) => indicator.derive === undefined)
+    const results = await refresh.refreshAll(
+      direct.map((indicator) => ({
+        indicatorId: indicator.id,
+        adapterId: indicator.source.adapter,
+        seriesRef: indicator.source.seriesRef,
+        params: indicator.source.params,
+        range,
+        freq: indicator.freq,
+        runtime: { ...runtime, force: request.force === true },
+      })),
+    )
+    const byId = new Map(direct.map((indicator, index) => [indicator.id, results[index]]))
+
+    // Derived indicators are computed from their operands' *raw* series, then
+    // everything goes through the same display pipeline.
+    const displayById = new Map()
+    for (const indicator of selected) {
+      if (indicator.derive !== undefined) continue
+      const series = byId.get(indicator.id)
+      if (series === undefined || series.status === 'error') {
+        displayById.set(indicator.id, { raw: series, points: [], stats: undefined, status: series?.status ?? 'error' })
+        continue
+      }
+      displayById.set(indicator.id, { raw: series, ...displaySeries(indicator, series), status: series.status })
+    }
+
+    for (const indicator of selected.filter((entry) => entry.derive !== undefined)) {
+      const seriesById = {}
+      for (const operand of indicator.derive.operands) {
+        const operandSeries = byId.get(operand)
+        if (operandSeries?.points?.length) seriesById[operand] = operandSeries.points
+      }
+      const derived = applyDerive(indicator.derive, seriesById)
+      if (derived.points.length === 0) {
+        // A derived series that cannot be computed still carries the provenance
+        // of its operands: the card must stay traceable, and the reason has to
+        // explain itself rather than showing an empty card.
+        const operandRef = byId.get(indicator.derive.operands.find((operand) => byId.get(operand)?.sourceRef !== undefined))?.sourceRef
+        displayById.set(indicator.id, {
+          raw: operandRef === undefined
+            ? undefined
+            : {
+                sourceRef: { ...operandRef, seriesRef: indicator.derive.operands.join('+'), label: `${operandRef.label}（现算）` },
+              },
+          points: [],
+          stats: undefined,
+          status: 'missing',
+          derived: true,
+          reason: derived.reason,
+          missing: derived.missing,
+        })
+        continue
+      }
+      // Provenance for a derived series is the provenance of its first operand,
+      // labelled so a reader knows the number was computed here rather than
+      // published upstream.
+      const operandRef = byId.get(indicator.derive.operands[0])?.sourceRef
+      const sourceRef = operandRef === undefined
+        ? undefined
+        : { ...operandRef, seriesRef: indicator.derive.operands.join('+'), label: `${operandRef.label}（现算）` }
+      const series = {
+        adapterId: 'derived',
+        seriesRef: indicator.derive.operands.join('+'),
+        points: derived.points,
+        meta: { name: indicator.label.zh, freq: indicator.freq },
+        fetchedAt: clock.now().toISOString(),
+        sourceRef,
+      }
+      displayById.set(indicator.id, { raw: series, ...displaySeries(indicator, series), status: 'fresh', derived: true })
+    }
+
+    const statusById = {}
+    const seriesForRules = {}
+    for (const [id, entry] of displayById) {
+      statusById[id] = entry.status
+      if (entry.points.length > 0) seriesForRules[id] = { points: entry.points, stats: entry.stats }
+    }
+
+    const { hits, errors: ruleErrors } = runRules({ today: clock.today(), seriesById: seriesForRules, catalogById })
+    const noteworthy = buildNoteworthy(hits, {
+      catalogById,
+      statusById,
+      limit: request.limit ?? 5,
+      minScore: 1,
+    })
+
+    const metrics = selected.map((indicator) => {
+      const entry = displayById.get(indicator.id)
+      const windowPoints = entry.points
+      const ref = entry.raw?.sourceRef
+      return {
+        indicatorId: indicator.id,
+        group: indicator.group,
+        label: indicator.label,
+        unit: indicator.unit,
+        freq: indicator.freq,
+        seasonal: indicator.seasonal,
+        importance: indicator.importance,
+        display: indicator.display,
+        notes: indicator.notes,
+        tags: indicator.tags ?? [],
+        latest: entry.stats?.latest,
+        latestAt: entry.stats?.latestAt,
+        changeAbs: entry.stats?.changeAbs,
+        changePct: entry.stats?.changePct,
+        yoy: entry.stats?.yoy,
+        sparkline: sparkline(windowPoints),
+        pointCount: windowPoints.length,
+        status: entry.status,
+        lastSuccessAt: entry.raw?.lastSuccessAt,
+        errorKind: entry.raw?.error?.kind,
+        errorDetail: entry.raw?.error?.detail,
+        sourceRef: ref,
+        derived: entry.derived === true,
+        hits: hits.filter((hit) => hit.indicatorId === indicator.id).map((hit) => ({ ruleId: hit.ruleId, score: hit.score, reason: hit.reason, evidence: hit.evidence })),
+        score: noteworthy.find((item) => item.indicatorId === indicator.id)?.score ?? 0,
+      }
+    })
+
+    const sources = {}
+    for (const [id, entry] of byId) {
+      const indicator = catalogById[id]
+      const key = indicator?.source?.adapter ?? 'unknown'
+      const bucket = sources[key] ?? { adapterId: key, ok: 0, degraded: 0, failed: 0, lastSuccessAt: undefined }
+      if (entry.status === 'error') bucket.failed += 1
+      else if (entry.status === 'stale') bucket.degraded += 1
+      else bucket.ok += 1
+      if (entry.lastSuccessAt !== undefined && (bucket.lastSuccessAt === undefined || entry.lastSuccessAt > bucket.lastSuccessAt)) {
+        bucket.lastSuccessAt = entry.lastSuccessAt
+      }
+      sources[key] = bucket
+    }
+
+    // Two indicators can share one series (CPI level and CPI yoy), so the error
+    // carries the indicator id from the request rather than being reverse-looked
+    // up by series ref. Identical failures are reported once: the panel shows a
+    // health list, not a request log.
+    const healthErrors = []
+    const seenErrors = new Set()
+    for (const entry of results) {
+      if (entry.status !== 'error') continue
+      const key = `${entry.indicatorId ?? ''}|${entry.error?.adapterId ?? ''}|${entry.error?.seriesRef ?? ''}`
+      if (seenErrors.has(key)) continue
+      seenErrors.add(key)
+      healthErrors.push({ indicatorId: entry.indicatorId, ...entry.error })
+    }
+    for (const error of ruleErrors) log('insight rule failed', error)
+
+    return {
+      range,
+      metrics,
+      noteworthy: noteworthy.map((entry) => ({
+        indicatorId: entry.indicatorId,
+        label: catalogById[entry.indicatorId]?.label,
+        unit: catalogById[entry.indicatorId]?.unit,
+        score: entry.score,
+        importance: entry.importance,
+        latestAt: entry.latestAt,
+        status: entry.status,
+        reasons: entry.hits.map((hit) => ({ ruleId: hit.ruleId, reason: hit.reason, evidence: hit.evidence })),
+        sourceRef: metrics.find((metric) => metric.indicatorId === entry.indicatorId)?.sourceRef,
+      })),
+      sources,
+      errors: healthErrors,
+      generatedAt: clock.now().toISOString(),
+      degraded: metrics.some((metric) => metric.status !== 'fresh'),
+    }
+  }
+
+  /**
+   * The raw (pre-transform) refresh result for one indicator, used by the AI
+   * layer to render a digest. Derived indicators resolve through their operands.
+   *
+   * @param {string} indicatorId - indicator id.
+   * @param {{ preset: string, from: string, to: string }} range - resolved range.
+   * @returns {Promise<{ points: Array<{ t: string, v: number }>, stats?: object, sourceRef?: object, status: string }|undefined>} series.
+   */
+  async function seriesFor(indicatorId, range) {
+    const indicator = catalogById[indicatorId]
+    if (indicator === undefined) return undefined
+    if (indicator.derive !== undefined) {
+      const seriesById = {}
+      for (const operand of indicator.derive.operands) {
+        const operandDef = catalogById[operand]
+        if (operandDef?.derive !== undefined) continue
+        const result = await refresh.refresh({
+          adapterId: operandDef.source.adapter,
+          seriesRef: operandDef.source.seriesRef,
+          params: operandDef.source.params,
+          range,
+          freq: operandDef.freq,
+          runtime,
+        })
+        if (result.points?.length) seriesById[operand] = result.points
+      }
+      const derived = applyDerive(indicator.derive, seriesById)
+      const operandRef = catalogById[indicator.derive.operands[0]]?.source
+      return {
+        points: derived.points,
+        stats: derived.points.length === 0 ? undefined : describeReturned(indicator, displaySeries(indicator, { points: derived.points }), range),
+        sourceRef: operandRef === undefined ? undefined : { ...operandRef, seriesRef: indicator.derive.operands.join('+'), label: '现算' },
+        status: derived.points.length > 0 ? 'fresh' : 'missing',
+      }
+    }
+    const result = await refresh.refresh({
+      adapterId: indicator.source.adapter,
+      seriesRef: indicator.source.seriesRef,
+      params: indicator.source.params,
+      range,
+      freq: indicator.freq,
+      runtime,
+    })
+    const display = displaySeries(indicator, result)
+    return { points: display.points, stats: describeReturned(indicator, display, range), sourceRef: result.sourceRef, status: result.status }
+  }
+
+  return { overview, rangeFor, displaySeries, sparkline, seriesFor }
+}

package/src/app/propose-indicator.js

@@ -0,0 +1,122 @@
+/**
+ * Conversational indicator addition (docs/06 §4).
+ *
+ * The gateway proposes, but **this module decides**. A candidate is only
+ * accepted after it passes the schema validator *and* its series actually
+ * fetches: a fabricated 'seriesRef' fails the fetch and lands in 'unsupported'
+ * with the failure reason. That is the mechanical guarantee behind "the AI may
+ * never invent a series id".
+ *
+ * @module app/propose-indicator
+ */
+import { validateIndicatorDef } from '../core/types.js'
+import { searchIndicators } from '../core/indicators/resolve.js'
+import { adapterIds } from '../sources/registry.js'
+
+/**
+ * Create the propose use case.
+ *
+ * @param {object} deps - dependencies.
+ * @param {object} deps.gateway - 'AiGateway' (its 'propose' is used).
+ * @param {Record<string, object>} deps.catalogById - existing catalog.
+ * @param {object} deps.refresh - refresh service, used to verify a series fetches.
+ * @param {import('../ports/clock.js').Clock} deps.clock - clock.
+ * @param {object} deps.runtime - runtime deps for adapters.
+ * @param {(adapterId: string) => boolean} deps.hasAdapter - adapter registry lookup.
+ * @param {(msg: string, meta?: object) => void} [deps.log] - logger.
+ * @returns {object} propose use case.
+ */
+export function createProposeUseCase({ gateway, catalogById, refresh, clock, runtime, hasAdapter, log = () => {} }) {
+  /**
+   * Propose indicator definitions for a natural-language request.
+   *
+   * @param {{ text: string, verify?: boolean }} request - request.
+   * @returns {Promise<object>} '{ candidates, unsupported, rejected, mode }'.
+   */
+  async function propose(request = {}) {
+    const { text, verify = true } = request
+    // The search runs here so both gateway shapes work: the deterministic one
+    // takes catalog matches, and an LLM gateway only needs the text.
+    const search = searchIndicators(text)
+    const proposal = (await gateway.propose({
+      request: text,
+      supportedAdapters: adapterIds(),
+      knownIds: Object.keys(catalogById),
+      matches: search.matches,
+    })) ?? { candidates: [], unsupported: [] }
+
+    const rejected = []
+    const candidates = []
+    const unsupported = [...(proposal.unsupported ?? [])]
+
+    for (const candidate of proposal.candidates ?? []) {
+      // An echo of a catalog entry is already schema-valid (the catalog suite
+      // enforces that), so only genuinely new definitions are re-validated here.
+      const validation = candidate.fromCatalog === true ? { ok: true, errors: [] } : validateIndicatorDef(candidate)
+      if (!validation.ok) {
+        rejected.push({ candidate, reason: validation.errors.join('; '), stage: 'schema' })
+        continue
+      }
+      if (!hasAdapter(candidate.source.adapter)) {
+        rejected.push({ candidate, reason: `unknown adapter ${candidate.source.adapter}`, stage: 'registry' })
+        unsupported.push({
+          request: candidate.label?.zh ?? candidate.id,
+          reason: `数据源适配器 ${candidate.source.adapter} 未注册`,
+          alternatives: ['换用已注册的数据源', '新增该数据源适配器后再试'],
+        })
+        continue
+      }
+
+      if (verify) {
+        // The anti-hallucination gate: an invented seriesRef cannot fetch.
+        const range = { preset: '1Y', from: clock.today(), to: clock.today() }
+        const probe = await refresh.refresh({
+          adapterId: candidate.source.adapter,
+          seriesRef: candidate.source.seriesRef,
+          params: candidate.source.params,
+          range,
+          freq: candidate.freq,
+          runtime: { ...runtime, force: true },
+        })
+        if (probe.status !== 'fresh' && probe.status !== 'stale') {
+          rejected.push({ candidate, reason: `series did not fetch: ${probe.error?.detail ?? probe.status}`, stage: 'fetch' })
+          unsupported.push({
+            request: candidate.label?.zh ?? candidate.id,
+            reason: `无法从 ${candidate.source.adapter} 取到 seriesRef「${candidate.source.seriesRef}」的数据（${probe.error?.kind ?? probe.status}）`,
+            alternatives: ['确认上游是否存在该序列', '改用已有指标作为代理'],
+          })
+          continue
+        }
+        candidates.push({
+          ...candidate,
+          conflict: catalogById[candidate.id] !== undefined,
+          verified: { points: probe.points.length, latestAt: probe.latestAt, status: probe.status },
+        })
+      } else {
+        candidates.push({ ...candidate, conflict: catalogById[candidate.id] !== undefined })
+      }
+    }
+
+    // A rejected candidate is an answer the user needs: say what was attempted and
+    // why it could not be added, instead of returning an empty result.
+    for (const entry of rejected) {
+      const label = entry.candidate?.label?.zh ?? entry.candidate?.id ?? '（未命名）'
+      unsupported.push({
+        request: label,
+        reason:
+          entry.stage === 'fetch'
+            ? `无法从 ${entry.candidate?.source?.adapter ?? '数据源'} 取到 seriesRef「${entry.candidate?.source?.seriesRef ?? ''}」的数据，因此没有加入。`
+            : `候选不合规（${entry.stage}）：${entry.reason}`,
+        alternatives:
+          entry.stage === 'fetch'
+            ? ['确认上游是否存在该序列', '改用已收录的同类指标作为代理']
+            : ['在指标目录中手动检索已有指标'],
+      })
+    }
+
+    log('propose finished', { candidates: candidates.length, rejected: rejected.length, unsupported: unsupported.length })
+    return { candidates, unsupported, rejected, mode: proposal.mode ?? gateway.describe?.().mode }
+  }
+
+  return { propose }
+}