dsh-plugin-show-me-data 0.1.0

package/lib/app/ai-validate.js

@@ -0,0 +1,284 @@
+/**
+ * Output validation — the enforcement point that turns "AI explanation" into a
+ * verifiable feature (docs/06 §3).
+ *
+ * Three checks, in order of severity:
+ * 1. every '[id@date, value]' citation must match a real digest point;
+ * 2. every number in the text must be traceable to the digest (or be a
+ *    whitelisted shape such as a year, a section ordinal or a percentage sign);
+ * 3. the answer must not claim the future or give advice.
+ *
+ * A violation never reaches the user: the caller retries once with the feedback,
+ * then degrades to the deterministic summary.
+ *
+ * @module app/ai-validate
+ */
+
+/** Citation syntax the model is told to use: '[id@date, value]'. */
+export const CITATION_RE = /\[([a-z0-9]+(?:\.[a-z0-9]+)+)@(\d{4}-\d{2}-\d{2})\s*,\s*(-?\d+(?:\.\d+)?)\s*\]/g
+
+/** Numbers that are allowed without a data source, with the reason why. */
+export const NUMBER_WHITELIST = [
+  { pattern: /^(19|20)\d{2}$/, reason: 'year' },
+  { pattern: /^\d{4}-\d{2}-\d{2}$/, reason: 'date' },
+  { pattern: /^(0|1|2|3|4|5|6|7|8|9|10)$/, reason: 'small ordinal' },
+  { pattern: /^(100|12|4|3|2|1)$/, reason: 'structural constant (percent/month/quarter counts)' },
+]
+
+/**
+ * Whether a number is the difference between two published values.
+ *
+ * "从 4.5% 到 4.1%，两端相差 0.4 个百分点" is the whole point of a data panel,
+ * and both observations are cited in the same sentence. Refusing it degraded
+ * correct answers; accepting it still refuses a number that appears nowhere in
+ * the data (a valuation multiple, a policy rate the panel never fetched).
+ *
+ * @param {number} value - number found in the text.
+ * @param {number[]} candidates - published numbers.
+ * @param {number} [tolerance] - rounding tolerance for the difference.
+ * @returns {boolean} whether it is a difference of two published values.
+ */
+export function isPublishedDifference(value, candidates) {
+  // Tolerance follows the precision the digest prints: two readings quoted to
+  // one decimal are subtracted to one decimal by anyone reading them, so the
+  // allowance is half of that last digit and no more. A wider window would wave
+  // through a number that merely happens to sit near *some* difference.
+  const tolerance = 0.05 / 10 ** Math.max(0, decimalsOf(value) - 1)
+  for (const a of candidates) {
+    if (!Number.isFinite(a)) continue
+    for (const b of candidates) {
+      if (!Number.isFinite(b)) continue
+      if (Math.abs(Math.abs(a - b) - value) <= tolerance) return true
+    }
+  }
+  return false
+}
+
+/**
+ * Digits after the decimal point in the *written* form of a number.
+ *
+ * @param {number} value - number from the text.
+ * @returns {number} decimal places (0 for an integer).
+ */
+function decimalsOf(value) {
+  const text = String(value)
+  const dot = text.indexOf('.')
+  return dot === -1 ? 0 : text.length - dot - 1
+}
+
+/** Phrases that assert a prediction or investment advice. */
+export const FORBIDDEN_PATTERNS = [
+  { pattern: /必然|一定会|肯定会|毫无疑问/, reason: 'absolute wording' },
+  { pattern: /建议(买入|卖出|加仓|减仓|做多|做空)|买入信号|卖出信号/, reason: 'investment advice' },
+  // A modal future marker followed by a direction verb. Deliberately not
+  // date-anchored: in practice any `会/将 + 涨跌` in a data panel's output is a
+  // forecast, and requiring a nearby date made the check miss the common
+  // phrasing `明天会上涨`.
+  { pattern: /(?:会|将)\s*(?:继续)?\s*(?:上涨|涨|下跌|跌|上行|下行|走高|走低|突破|跌破)/, reason: 'price prediction' },
+  { pattern: /目标价|看到\d+点|涨到\d+|跌到\d+/, reason: 'price target' },
+]
+
+/**
+ * One digest entry flattened for validation.
+ *
+ * @typedef {Object} ValidationData
+ * @property {Record<string, { values: number[], dates: string[], stats: number[] }>} byIndicator
+ * @property {number[]} counts - observation counts the digest states about itself.
+ */
+
+/**
+ * Build the validation index from digest entries.
+ *
+ * @param {Array<{ indicator: object, stats?: object, points?: Array<{ t: string, v: number }> }>} entries - digest entries.
+ * @returns {ValidationData} validation data.
+ */
+export function buildValidationData(entries) {
+  const byIndicator = {}
+  // Counts the digest itself publishes ("points=13 missing=2"). They describe the
+  // series rather than measure it, so a model restating them is not inventing a
+  // number — refusing them would penalise it for reading our own output.
+  const counts = []
+  for (const entry of entries ?? []) {
+    const id = entry?.indicator?.id
+    if (typeof id !== 'string') continue
+    const bucket = byIndicator[id] ?? { values: [], dates: [], stats: [] }
+    for (const point of entry.points ?? []) {
+      bucket.values.push(point.v)
+      bucket.dates.push(point.t)
+    }
+    const stats = entry.stats ?? {}
+    for (const key of ['count', 'missingCount']) {
+      if (typeof stats[key] === 'number' && Number.isFinite(stats[key])) counts.push(stats[key])
+    }
+    counts.push((entry.points ?? []).length)
+    for (const key of ['latest', 'prev', 'changeAbs', 'changePct', 'yoy', 'mom', 'mean', 'min', 'max', 'stdDev', 'slope', 'zScoreLatestChange', 'percentile']) {
+      if (typeof stats[key] === 'number' && Number.isFinite(stats[key])) bucket.stats.push(stats[key])
+    }
+    if (typeof stats.latest === 'number') bucket.values.push(stats.latest)
+    if (typeof stats.latestAt === 'string') bucket.dates.push(stats.latestAt)
+    byIndicator[id] = bucket
+  }
+  return { byIndicator, counts }
+}
+
+/**
+ * Whether a number appears in the digest, within a display rounding tolerance.
+ *
+ * @param {number} value - the number found in the text.
+ * @param {number[]} candidates - digest numbers.
+ * @returns {boolean} whether it is traceable.
+ */
+export function matchesData(value, candidates) {
+  // Tolerance is deliberately tight: a number the model invented must not pass
+  // just because it is "close enough" to a real observation.
+  const tolerance = 1e-6
+  for (const candidate of candidates) {
+    if (!Number.isFinite(candidate)) continue
+    if (Math.abs(candidate - value) <= tolerance) return true
+    // The model is asked to round to `decimals`; accept any rounding of the value.
+    for (const decimals of [0, 1, 2, 3, 4]) {
+      if (Number(candidate.toFixed(decimals)) === value) return true
+    }
+    // Percentage points written without the sign, and unit-scaled forms
+    // (e.g. a percentage reported in decimal form).
+    if (Math.abs(candidate * 100 - value) <= tolerance) return true
+    if (Math.abs(candidate / 100 - value) <= tolerance) return true
+  }
+  return false
+}
+
+/**
+ * Extract every number literal from a text, with its position.
+ *
+ * @param {string} text - answer text.
+ * @returns {Array<{ raw: string, value: number, index: number }>} numbers.
+ */
+export function extractNumbers(text) {
+  const found = []
+  // Strip citations (they are validated separately) and calendar dates: the
+  // digits inside `2026-08-01` are not claims, and matching them produced
+  // nonsense violations like `-08`.
+  const cleaned = String(text ?? '')
+    .replace(CITATION_RE, ' ')
+    .replace(/\d{4}-\d{2}-\d{2}/g, ' ')
+    .replace(/\d{4}-\d{2}(?!\d)/g, ' ')
+  for (const match of cleaned.matchAll(/(?<![\d-])-?\d+(?:\.\d+)?/g)) {
+    found.push({ raw: match[0], value: Number(match[0]), index: match.index ?? 0 })
+  }
+  return found
+}
+
+/**
+ * Validate one answer against the digest.
+ *
+ * @param {object} input - validation input.
+ * @param {string} input.text - the model's answer.
+ * @param {ValidationData} input.data - digest index.
+ * @param {boolean} [input.allowEmptyCitations] - permit "数据不足" answers without citations.
+ * @param {string} [input.echoOf] - text the answer may legitimately contain verbatim (the user's question).
+ * @returns {{ ok: boolean, violations: Array<{ kind: string, detail: string, raw?: string }>, usedPoints: Array<{ indicatorId: string, t: string, v: number }>, insufficient: string|undefined }} verdict.
+ */
+export function validateAnswer({ text, data, allowEmptyCitations = true, echoOf, digestText }) {
+  const violations = []
+  const usedPoints = []
+  const source = typeof text === 'string' ? text : ''
+  const knownIds = new Set(Object.keys(data?.byIndicator ?? {}))
+
+  // An empty answer is never acceptable: the caller degrades to the
+  // deterministic summary instead of showing the user a blank panel.
+  if (source.trim() === '') {
+    return {
+      ok: false,
+      violations: [{ kind: 'empty-answer', detail: '模型返回了空回答' }],
+      usedPoints: [],
+      insufficient: undefined,
+    }
+  }
+
+  // 1) citations
+  for (const match of source.matchAll(CITATION_RE)) {
+    const [, indicatorId, date, rawValue] = match
+    const bucket = data?.byIndicator?.[indicatorId]
+    if (bucket === undefined) {
+      violations.push({ kind: 'unknown-indicator', detail: `引用不存在的指标 ${indicatorId}`, raw: match[0] })
+      continue
+    }
+    if (!bucket.dates.includes(date)) {
+      violations.push({ kind: 'unknown-date', detail: `${indicatorId} 在 ${date} 没有观测`, raw: match[0] })
+      continue
+    }
+    const value = Number(rawValue)
+    if (!matchesData(value, bucket.values)) {
+      violations.push({ kind: 'value-mismatch', detail: `${indicatorId}@${date} 的值 ${value} 与数据不符`, raw: match[0] })
+      continue
+    }
+    usedPoints.push({ indicatorId, t: date, v: value })
+  }
+
+  // 2) unsourced numbers
+  //
+  // "Sourced" means the digest shows it. Statistics and observations are the
+  // obvious part, but a label ("标普 500"), a field name and a computed delta all
+  // print there too, and a model restating any of them was being accused of
+  // inventing a number — which degraded perfectly good answers to the
+  // deterministic summary. Everything the digest prints is therefore traceable
+  // by definition, so the digest's own text is scanned as one more source.
+  const printed = extractNumbers(typeof digestText === 'string' ? digestText : '')
+  // Measurements only. `counts` and the digest text carry structural numbers
+  // ("points=250", a label's "500"), and differencing those against a reading
+  // would let an invented number through as "4.1 - 3".
+  const measurements = Object.values(data?.byIndicator ?? {}).flatMap((bucket) => [...bucket.values, ...bucket.stats])
+  const allNumbers = [...measurements, ...(data?.counts ?? []), ...printed.map((number) => number.value)]
+  for (const number of extractNumbers(source)) {
+    if (NUMBER_WHITELIST.some((entry) => entry.pattern.test(number.raw))) continue
+    if (matchesData(number.value, allNumbers)) continue
+    // Rounding tolerance, not a free pass: a reader comparing two published
+    // points does exactly this subtraction, and a published difference is
+    // arithmetic on our data rather than a fabricated observation.
+    if (isPublishedDifference(number.value, measurements)) continue
+    violations.push({ kind: 'unsourced-number', detail: `数字 ${number.raw} 无法在数据中找到来源`, raw: number.raw })
+  }
+
+  // 3) predictions and advice. A phrase the user themselves wrote is echoed by
+  // the answer ("数据不足：无法回答「明天油价会涨吗？」"), which is not the
+  // answer asserting a forecast — so the question's own wording is excused.
+  const echo = typeof echoOf === 'string' ? echoOf : ''
+  const withoutEcho = echo === '' ? source : source.split(echo).join(' ')
+  for (const rule of FORBIDDEN_PATTERNS) {
+    const match = rule.pattern.exec(withoutEcho)
+    if (match !== null) violations.push({ kind: 'forbidden-wording', detail: `${rule.reason}：${match[0]}`, raw: match[0] })
+  }
+
+  const insufficient = /数据不足/.test(source) ? source.match(/数据不足[^。\n]*/)?.[0] ?? '数据不足' : undefined
+  if (usedPoints.length === 0 && insufficient === undefined && !allowEmptyCitations) {
+    violations.push({ kind: 'no-citations', detail: '回答没有引用任何数据点' })
+  }
+  if (usedPoints.length === 0 && insufficient === undefined && knownIds.size > 0 && source.trim() !== '') {
+    violations.push({ kind: 'no-citations', detail: '回答没有引用任何数据点，也没有说明数据不足' })
+  }
+
+  return { ok: violations.length === 0, violations, usedPoints, insufficient }
+}
+
+/**
+ * Decide what to do with a verdict (docs/06 §3 rule 3).
+ *
+ * @param {{ ok: boolean }} verdict - validation verdict.
+ * @param {number} attempt - how many attempts have been made (1-based).
+ * @param {number} [maxAttempts] - retry budget.
+ * @returns {'accept'|'retry'|'degrade'} action.
+ */
+export function decideAction(verdict, attempt, maxAttempts = 2) {
+  if (verdict.ok) return 'accept'
+  return attempt < maxAttempts ? 'retry' : 'degrade'
+}
+
+/**
+ * Human-readable violation list for the retry prompt.
+ *
+ * @param {Array<{ detail: string }>} violations - violations.
+ * @returns {string[]} lines.
+ */
+export function violationLines(violations) {
+  return (violations ?? []).map((entry) => entry.detail)
+}