dsh-plugin-show-me-data 0.1.0

package/lib/sources/fred.js

@@ -0,0 +1,207 @@
+/**
+ * FRED CSV adapter — the backbone source (docs/04 §1).
+ *
+ * 'fredgraph.csv' needs no API key and returns a two-column CSV whose first
+ * column is always 'observation_date'; a missing observation is a literal '.'
+ * and must be skipped, never read as zero.
+ *
+ * @module sources/fred
+ */
+import { SourceError } from '../core/types.js'
+import { addDays, addMonths, daysBetween } from '../core/time/range.js'
+import { parseNumber } from '../core/stats/series.js'
+import { getText } from './http.js'
+
+/** @type {string} */
+export const id = 'fred'
+
+/** @type {string} */
+export const label = 'FRED'
+
+/** @type {Array<{ kinds: string[], frequencies: string[] }>} */
+export const capabilities = [
+  { kinds: ['macro', 'rates', 'equity', 'commodity', 'labor'], frequencies: ['daily', 'weekly', 'monthly', 'quarterly', 'annual'] },
+]
+
+/** How far before 'from' the adapter requests history, so annual comparisons work. */
+const LOOKBACK_MONTHS = 13
+
+/** Slugs that mean "the whole series", independent of frequency. */
+const MAX_LOOKBACK_MONTHS = 12 * 60
+
+/**
+ * Build the CSV URL for one series.
+ *
+ * 'cosd' is mandatory in practice: without it 'GDPC1' returns its full history
+ * since 1947 (docs/04 §1).
+ *
+ * @param {string} seriesRef - FRED series id.
+ * @param {{ from: string, to: string }} range - requested window.
+ * @param {{ preset?: string }} [options] - range preset, when known.
+ * @returns {string} request URL.
+ */
+export function buildUrl(seriesRef, range, { preset } = {}) {
+  const cosd = preset === 'MAX' ? addMonths(range.from, -MAX_LOOKBACK_MONTHS) : addMonths(range.from, -LOOKBACK_MONTHS)
+  return `https://fred.stlouisfed.org/graph/fredgraph.csv?id=${encodeURIComponent(seriesRef)}&cosd=${cosd}&coed=${range.to}`
+}
+
+/**
+ * Human-readable landing page plus the machine-readable CSV URL (docs/03 §1.1).
+ *
+ * @param {string} seriesRef - FRED series id.
+ * @returns {{ adapterId: string, url: string, apiUrl: string, seriesRef: string, label: string }} source reference.
+ */
+export function sourceRef(seriesRef) {
+  return {
+    adapterId: id,
+    url: `https://fred.stlouisfed.org/series/${encodeURIComponent(seriesRef)}`,
+    apiUrl: `https://fred.stlouisfed.org/graph/fredgraph.csv?id=${encodeURIComponent(seriesRef)}`,
+    seriesRef,
+    label,
+  }
+}
+
+/**
+ * Parse a FRED CSV body.
+ *
+ * @param {string} body - CSV text.
+ * @param {string} seriesRef - expected series id (the header's second column).
+ * @param {{ adapterId?: string }} [options] - owning adapter id for errors.
+ * @returns {Array<{ t: string, v: number }>} ascending points.
+ */
+export function parseCsv(body, seriesRef, { adapterId = id } = {}) {
+  const lines = body.trim().split(/\r?\n/)
+  if (lines.length === 0 || lines[0].trim() === '') {
+    throw new SourceError({ kind: 'empty', adapterId, seriesRef, detail: 'CSV body has no header row' })
+  }
+  const header = lines[0].split(',')
+  if (header.length < 2) {
+    throw new SourceError({ kind: 'parse', adapterId, seriesRef, detail: `expected two CSV columns, got ${JSON.stringify(lines[0])}` })
+  }
+  if (header[0].trim() !== 'observation_date') {
+    throw new SourceError({
+      kind: 'parse',
+      adapterId,
+      seriesRef,
+      detail: `unexpected first column ${JSON.stringify(header[0])}; expected observation_date`,
+    })
+  }
+  const points = []
+  for (const line of lines.slice(1)) {
+    if (line.trim() === '') continue
+    const [date, raw] = line.split(',')
+    const value = parseNumber(raw)
+    if (value === undefined) continue // '.' and friends: missing, not zero
+    if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) continue
+    points.push({ t: date, v: value })
+  }
+  if (points.length === 0) {
+    throw new SourceError({
+      kind: 'empty',
+      adapterId,
+      seriesRef,
+      detail: 'the CSV parsed but contained no usable observations',
+    })
+  }
+  points.sort((a, b) => (a.t < b.t ? -1 : a.t > b.t ? 1 : 0))
+  return points
+}
+
+/**
+ * Fetch and normalize one FRED series.
+ *
+ * @param {{ seriesRef: string, range: { from: string, to: string }, preset?: string }} req - request.
+ * @param {{ fetch: Function, clock: { now: () => Date }, signal?: AbortSignal }} deps - injected dependencies.
+ * @returns {Promise<object>} normalized 'RawSeries'.
+ */
+export async function fetchSeries(req, deps) {
+  const { seriesRef, range, preset } = req
+  const url = buildUrl(seriesRef, range, { preset })
+  const { status, body } = await getText({
+    fetch: deps.fetch,
+    url,
+    adapterId: id,
+    seriesRef,
+    signal: deps.signal,
+    allowHttpError: true,
+  })
+
+  if (status >= 400 && status < 500) {
+    // FRED answers an unknown series id with a 404 HTML page, not error JSON:
+    // that is a wrong id (never retryable), not a transient failure.
+    throw new SourceError({
+      kind: 'unsupported',
+      adapterId: id,
+      seriesRef,
+      detail: `FRED returned HTTP ${status} for ${seriesRef}; the series id does not exist`,
+      httpStatus: status,
+    })
+  }
+  if (status >= 500) {
+    throw new SourceError({
+      kind: 'http',
+      adapterId: id,
+      seriesRef,
+      detail: `FRED is unavailable (HTTP ${status})`,
+      httpStatus: status,
+    })
+  }
+  if (body.trimStart().startsWith('<')) {
+    throw new SourceError({
+      kind: 'parse',
+      adapterId: id,
+      seriesRef,
+      detail: 'FRED returned an HTML page instead of CSV',
+      httpStatus: status,
+    })
+  }
+
+  const points = parseCsv(body, seriesRef)
+  return {
+    adapterId: id,
+    seriesRef,
+    points,
+    meta: {
+      // `fredgraph.csv` labels the column with the series id, not a human
+      // title; the catalog carries the human label, so the source name stays
+      // machine-checkable here.
+      name: seriesRef,
+      unit: undefined,
+      freq: inferFrequency(points),
+    },
+    fetchedAt: deps.clock.now().toISOString(),
+    sourceRef: sourceRef(seriesRef),
+    raw: { url, httpStatus: status, bytes: body.length },
+  }
+}
+
+/**
+ * Infer the observation cadence from the median gap — used for display only,
+ * the catalog's 'freq' is authoritative for TTL and staleness.
+ *
+ * @param {Array<{ t: string, v: number }>} points - ascending points.
+ * @returns {string} frequency id.
+ */
+export function inferFrequency(points) {
+  if (points.length < 3) return 'monthly'
+  const gaps = []
+  for (let i = 1; i < points.length; i += 1) gaps.push(daysBetween(points[i - 1].t, points[i].t))
+  gaps.sort((a, b) => a - b)
+  const median = gaps[Math.floor(gaps.length / 2)]
+  if (median <= 3) return 'daily'
+  if (median <= 14) return 'weekly'
+  if (median <= 45) return 'monthly'
+  if (median <= 120) return 'quarterly'
+  return 'annual'
+}
+
+/**
+ * The window actually covered by a request (kept separate so tests can assert
+ * the 'cosd' expansion without a network round trip).
+ *
+ * @param {{ from: string, to: string }} range - requested window.
+ * @returns {{ from: string, to: string }} expanded window.
+ */
+export function requestedWindow(range) {
+  return { from: addMonths(range.from, -LOOKBACK_MONTHS), to: addDays(range.to, 0) }
+}

package/lib/sources/http.js

@@ -0,0 +1,136 @@
+/**
+ * Shared HTTP plumbing for adapters — the only place a source touches the
+ * network (docs/03 §3.1). Adapters stay thin and pure over the text they get
+ * back, which is what keeps them fixture-replayable.
+ *
+ * @module sources/http
+ */
+import { SourceError } from '../core/types.js'
+
+/** Per-request timeout (docs/04 §8). */
+export const REQUEST_TIMEOUT_MS = 15_000
+
+/** Browser-ish user agent; several of these sources reject the default. */
+export const USER_AGENT = 'Mozilla/5.0 (compatible; dsh-plugin-show-me-data)'
+
+/**
+ * Compose the caller's cancellation signal with this request's own timeout.
+ *
+ * @param {AbortSignal|undefined} signal - caller signal.
+ * @param {number} timeoutMs - timeout budget.
+ * @returns {{ signal: AbortSignal, dispose: () => void }} combined signal and cleanup.
+ */
+function withTimeout(signal, timeoutMs) {
+  const timeout = new AbortController()
+  const timer = setTimeout(() => timeout.abort(new Error('timeout')), timeoutMs)
+  if (signal === undefined) return { signal: timeout.signal, dispose: () => clearTimeout(timer) }
+  const signals = [signal, timeout.signal]
+  const combined = typeof AbortSignal.any === 'function' ? AbortSignal.any(signals) : timeout.signal
+  return { signal: combined, dispose: () => clearTimeout(timer) }
+}
+
+/**
+ * Fetch one URL and return its body as text, classifying every failure into a
+ * 'SourceError' (docs/03 §1.7). Non-2xx responses are returned to the caller for
+ * 'parse'-level judgement only when 'allowHttpError' is set; otherwise they are
+ * 'http' errors.
+ *
+ * @param {object} options - request options.
+ * @param {Function} options.fetch - the 'fetch' implementation from 'deps.fetch'.
+ * @param {string} options.url - absolute URL.
+ * @param {string} options.adapterId - owning adapter.
+ * @param {string} options.seriesRef - requested series.
+ * @param {AbortSignal} [options.signal] - caller cancellation.
+ * @param {number} [options.timeoutMs] - timeout budget.
+ * @param {boolean} [options.allowHttpError] - return non-2xx bodies instead of throwing.
+ * @param {Record<string, string>} [options.headers] - extra headers.
+ * @returns {Promise<{ status: number, body: string, url: string }>} response body.
+ */
+export async function getText({
+  fetch,
+  url,
+  adapterId,
+  seriesRef,
+  signal,
+  timeoutMs = REQUEST_TIMEOUT_MS,
+  allowHttpError = false,
+  headers = {},
+}) {
+  if (typeof fetch !== 'function') {
+    throw new SourceError({
+      kind: 'network',
+      adapterId,
+      seriesRef,
+      detail: 'no fetch implementation was injected',
+    })
+  }
+  const { signal: combined, dispose } = withTimeout(signal, timeoutMs)
+  let response
+  try {
+    response = await fetch(url, { signal: combined, headers: { 'user-agent': USER_AGENT, ...headers } })
+  } catch (error) {
+    const aborted = error?.name === 'AbortError' || error?.name === 'TimeoutError'
+    throw new SourceError({
+      kind: 'network',
+      adapterId,
+      seriesRef,
+      detail: aborted ? `request aborted after ${timeoutMs}ms` : `${error?.name ?? 'Error'}: ${error?.message ?? error}`,
+    })
+  } finally {
+    dispose()
+  }
+
+  if (!response.ok && !allowHttpError) {
+    throw new SourceError({
+      kind: 'http',
+      adapterId,
+      seriesRef,
+      detail: `upstream responded ${response.status} ${response.statusText ?? ''}`.trim(),
+      httpStatus: response.status,
+    })
+  }
+
+  let body
+  try {
+    body = await response.text()
+  } catch (error) {
+    throw new SourceError({
+      kind: 'network',
+      adapterId,
+      seriesRef,
+      detail: `reading the response body failed: ${error?.message ?? error}`,
+    })
+  }
+
+  if (body.trim() === '') {
+    throw new SourceError({
+      kind: 'empty',
+      adapterId,
+      seriesRef,
+      detail: `upstream returned an empty body (HTTP ${response.status})`,
+      httpStatus: response.status,
+    })
+  }
+  return { status: response.status, body, url }
+}
+
+/**
+ * Parse JSON with a classified error instead of a raw 'SyntaxError'.
+ *
+ * @param {string} body - response body.
+ * @param {string} adapterId - owning adapter.
+ * @param {string} seriesRef - requested series.
+ * @returns {any} parsed value.
+ */
+export function parseJson(body, adapterId, seriesRef) {
+  try {
+    return JSON.parse(body)
+  } catch (error) {
+    throw new SourceError({
+      kind: 'parse',
+      adapterId,
+      seriesRef,
+      detail: `response is not JSON (${String(body.slice(0, 80)).replace(/\s+/g, ' ')}…)`,
+    })
+  }
+}

package/lib/sources/ohlc.js

@@ -0,0 +1,36 @@
+/**
+ * The one place an OHLC bar is accepted or rejected.
+ *
+ * Sources hand over four columns and are not guaranteed to agree with each
+ * other: the live Sina US payload contains a session whose low sits *above* its
+ * open. Forwarding that would draw a candle that cannot exist, and a panel that
+ * says "the low was higher than the open" has published a false number. The bar
+ * is therefore dropped while the close — a real observation — is kept, so the
+ * series loses detail and never loses a reading.
+ *
+ * @module sources/ohlc
+ */
+
+/**
+ * Whether four numbers form a coherent bar: high bounds every other price, low is
+ * bounded by every other price.
+ *
+ * @param {{ o: unknown, h: unknown, l: unknown, c: unknown }} bar - candidate bar.
+ * @returns {boolean} whether the bar is coherent and finite.
+ */
+export function coherentBar({ o, h, l, c }) {
+  if (!Number.isFinite(o) || !Number.isFinite(h) || !Number.isFinite(l) || !Number.isFinite(c)) return false
+  return h >= Math.max(o, c) && l <= Math.min(o, c)
+}
+
+/**
+ * Attach the bar to a point, or leave the point as a plain observation.
+ *
+ * @param {{ t: string, v: number }} point - point carrying the close as 'v'.
+ * @param {{ o: unknown, h: unknown, l: unknown }} bar - candidate bar columns.
+ * @returns {{ t: string, v: number, o?: number, h?: number, l?: number }} the point.
+ */
+export function withBar(point, bar) {
+  if (!coherentBar({ o: bar.o, h: bar.h, l: bar.l, c: point.v })) return point
+  return { ...point, o: bar.o, h: bar.h, l: bar.l }
+}

package/lib/sources/quote-cascade.js

@@ -0,0 +1,177 @@
+/**
+ * Quote cascade — try each quote source in order, remember which one works.
+ *
+ * The quote endpoints used here are unofficial and block from time to time
+ * (observed in one session: 东方财富 'push2his' resetting connections, then
+ * 腾讯 answering 501). A single hard binding would therefore take a whole group
+ * of equity cards down whenever one operator tightens its WAF, so this adapter
+ * keeps **the catalog binding stable** and moves the choice of upstream into
+ * runtime: each request tries the configured chain and returns the first
+ * success; the winner is remembered for later requests in the same process.
+ *
+ * The chain is data (one entry per upstream), and each entry is an ordinary
+ * adapter from the registry — so this file adds resilience without duplicating
+ * any parsing logic.
+ *
+ * @module sources/quote-cascade
+ */
+import { SourceError } from '../core/types.js'
+
+/** @type {string} */
+export const id = 'quote'
+
+/** @type {string} */
+export const label = '行情（多源）'
+
+/** @type {Array<{ kinds: string[], frequencies: string[] }>} */
+export const capabilities = [{ kinds: ['equity', 'bond'], frequencies: ['daily'] }]
+
+/**
+ * The upstream chain, in preference order.
+ *
+ * 'bindings' maps a cascade symbol to each upstream's own series reference.
+ *
+ * @type {Array<{ adapterId: string, bindings: Record<string, { seriesRef: string, params?: object }> }>}
+ */
+export const CHAIN = [
+  // Ordered by what actually answers from this network, most reliable first;
+  // the rest stay in the chain so a future block on one operator degrades
+  // instead of taking the cards down.
+  { adapterId: 'sina-cn', bindings: {} },
+  { adapterId: 'tencent', bindings: {} },
+  { adapterId: 'eastmoney-quote', bindings: {} },
+]
+
+/** Symbols the cascade serves, with their per-upstream references. */
+export const SYMBOLS = {
+  sse: { zh: '上证指数', 'sina-cn': 'sh000001', tencent: 'sh000001', 'eastmoney-quote': '1.000001' },
+  szse: { zh: '深证成指', 'sina-cn': 'sz399001', tencent: 'sz399001', 'eastmoney-quote': '0.399001' },
+  csi300: { zh: '沪深 300', 'sina-cn': 'sh000300', tencent: 'sh000300', 'eastmoney-quote': '1.000300' },
+  chinext: { zh: '创业板指', 'sina-cn': 'sz399006', tencent: 'sz399006', 'eastmoney-quote': '0.399006' },
+  govtbond: { zh: '上证国债指数', 'sina-cn': 'sh000012', tencent: 'sh000012', 'eastmoney-quote': '1.000012' },
+  corpbond: { zh: '上证企债指数', 'sina-cn': 'sh000013', tencent: 'sh000013', 'eastmoney-quote': '1.000013' },
+  dividend: { zh: '上证红利指数', 'sina-cn': 'sh000015', tencent: 'sh000015', 'eastmoney-quote': '1.000015' },
+}
+
+/**
+ * The cascade can re-sample: whichever upstream answers, it takes a bar size.
+ *
+ * The panel sends the size as a neutral name because members disagree on the
+ * parameter ('klt' vs 'scale'), and only this adapter knows which member will be
+ * tried.
+ */
+export const barSizes = true
+
+/** How each member spells a bar size. */
+const SIZE_PARAMS = {
+  'eastmoney-quote': { day: 101, week: 102, month: 103 },
+  'sina-cn': { day: 240, week: 1200, month: 7200 },
+}
+
+/**
+ * Translate the neutral size into the member's own parameter.
+ *
+ * @param {string} adapterId - upstream adapter id.
+ * @param {{ size?: string }} [params] - request params.
+ * @returns {object|undefined} member params.
+ */
+function paramsFor(adapterId, params) {
+  if (params === undefined || params.size === undefined) return params
+  // `size` is the cascade's own vocabulary: it must not reach a member, which
+  // would then put it in that member's cache key and URL.
+  const { size, ...rest } = params
+  const mapped = SIZE_PARAMS[adapterId]?.[size]
+  if (mapped === undefined) return rest
+  return { ...rest, ...(adapterId === 'eastmoney-quote' ? { klt: mapped } : { scale: mapped }) }
+}
+
+/** The last upstream that worked, remembered per symbol for this process. */
+const preferred = new Map()
+
+/**
+ * Build the ordered upstream attempts for one symbol.
+ *
+ * @param {string} seriesRef - cascade symbol (e.g. 'sse').
+ * @returns {Array<{ adapterId: string, seriesRef: string, params?: object }>} attempts.
+ */
+export function attemptsFor(seriesRef) {
+  const symbol = SYMBOLS[seriesRef]
+  if (symbol === undefined) {
+    throw new SourceError({
+      kind: 'unsupported',
+      adapterId: id,
+      seriesRef,
+      detail: `unknown quote symbol ${JSON.stringify(seriesRef)}; known: ${Object.keys(SYMBOLS).join(', ')}`,
+    })
+  }
+  const attempted = CHAIN.map((entry) => ({
+    adapterId: entry.adapterId,
+    seriesRef: symbol[entry.adapterId],
+  })).filter((entry) => typeof entry.seriesRef === 'string')
+  // Start with whichever upstream answered last: a blocked operator rarely
+  // recovers within a session, and this avoids paying its timeout every refresh.
+  const winner = preferred.get(seriesRef)
+  if (winner === undefined) return attempted
+  return [...attempted].sort((a, b) => (a.adapterId === winner ? -1 : b.adapterId === winner ? 1 : 0))
+}
+
+/**
+ * Source reference for the upstream that is currently preferred.
+ *
+ * @param {string} seriesRef - cascade symbol.
+ * @param {object} [params] - catalog params.
+ * @param {(id: string, ref: string) => object|undefined} [resolve] - registry lookup.
+ * @returns {{ adapterId: string, url: string, seriesRef: string, label: string, apiUrl?: string }} source reference.
+ */
+export function sourceRef(seriesRef, params, resolve) {
+  const attempts = attemptsFor(seriesRef)
+  const first = attempts[0]
+  const upstream = resolve?.(first.adapterId, first.seriesRef)
+  if (upstream !== undefined) return upstream
+  return { adapterId: id, url: 'https://quote.eastmoney.com/', seriesRef, label }
+}
+
+/**
+ * Fetch one symbol, falling through the chain.
+ *
+ * @param {{ seriesRef: string, params?: object, range?: object, preset?: string }} req - request.
+ * @param {{ fetch: Function, clock: object, signal?: AbortSignal, fetchViaAdapter?: Function, resolveSourceRef?: Function }} deps - injected dependencies.
+ * @returns {Promise<object>} normalized 'RawSeries' from the first upstream that answers.
+ */
+export async function fetchSeries(req, deps) {
+  const attempts = attemptsFor(req.seriesRef)
+  const failures = []
+  for (const attempt of attempts) {
+    try {
+      const series = await deps.fetchViaAdapter(attempt.adapterId, { ...req, seriesRef: attempt.seriesRef, params: paramsFor(attempt.adapterId, req.params) }, deps)
+      preferred.set(req.seriesRef, attempt.adapterId)
+      return {
+        ...series,
+        // The series identity stays the cascade symbol so the cache key and the
+        // catalog binding do not move when the upstream changes.
+        seriesRef: req.seriesRef,
+        raw: { ...(series.raw ?? {}), via: attempt.adapterId, upstreamSeriesRef: attempt.seriesRef },
+      }
+    } catch (error) {
+      failures.push({ adapterId: attempt.adapterId, seriesRef: attempt.seriesRef, kind: error?.kind ?? 'unknown', detail: error?.detail ?? String(error?.message ?? error) })
+    }
+  }
+  throw new SourceError({
+    kind: 'http',
+    adapterId: id,
+    seriesRef: req.seriesRef,
+    detail: `every upstream failed: ${failures.map((failure) => `${failure.adapterId}=${failure.kind}`).join(', ')}`,
+    ...(failures[0]?.detail === undefined ? {} : { httpStatus: undefined }),
+  })
+}
+
+/**
+ * Forget which upstream worked (used by tests and by a manual 'data_refresh').
+ *
+ * @param {string} [seriesRef] - symbol, or all when omitted.
+ * @returns {void}
+ */
+export function resetPreference(seriesRef) {
+  if (seriesRef === undefined) preferred.clear()
+  else preferred.delete(seriesRef)
+}