dsh-plugin-show-me-data 0.1.0

package/lib/sources/eastmoney-macro.js

@@ -0,0 +1,197 @@
+/**
+ * 东方财富数据中心 adapter — Chinese macro reports (docs/04 §3).
+ *
+ * Two traps live here and both are covered by tests:
+ *
+ * 1. **Column naming is inverted for money supply**: in
+ *    'RPT_ECONOMY_CURRENCY_SUPPLY', 'BASIC_CURRENCY' is M2 (≈3.55 quadrillion)
+ *    and 'CURRENCY' is M1 (≈1.15 quadrillion) — the opposite of intuition. The
+ *    catalog must always name the column explicitly ('params.column').
+ * 2. **Rows arrive newest-first** and 'success:false' still returns HTTP 200,
+ *    so the body — not the status — decides.
+ *
+ * @module sources/eastmoney-macro
+ */
+import { SourceError } from '../core/types.js'
+import { parseNumber } from '../core/stats/series.js'
+import { getText, parseJson } from './http.js'
+
+/** @type {string} */
+export const id = 'eastmoney-macro'
+
+/** @type {string} */
+export const label = '东方财富·数据中心'
+
+/** @type {Array<{ kinds: string[], frequencies: string[] }>} */
+export const capabilities = [{ kinds: ['macro'], frequencies: ['monthly', 'quarterly'] }]
+
+/** Landing page per report, for the source badge (docs/04 §3). */
+const REPORT_PAGES = {
+  RPT_ECONOMY_CPI: 'https://data.eastmoney.com/cjsj/cpi.html',
+  RPT_ECONOMY_PPI: 'https://data.eastmoney.com/cjsj/ppi.html',
+  RPT_ECONOMY_PMI: 'https://data.eastmoney.com/cjsj/pmi.html',
+  RPT_ECONOMY_GDP: 'https://data.eastmoney.com/cjsj/gdp.html',
+  RPT_ECONOMY_CURRENCY_SUPPLY: 'https://data.eastmoney.com/cjsj/hbgyl.html',
+  // Customs publishes both a year-on-year percentage and the level; the level is
+  // what a chart needs, the percentage is only a cross-check.
+  RPT_ECONOMY_CUSTOMS: 'https://data.eastmoney.com/cjsj/hgjck.html',
+  // The 70-city house price survey: this report is **per city**, so the series
+  // identity includes the city, not just the column.
+  RPT_ECONOMY_HOUSE_PRICE: 'https://data.eastmoney.com/cjsj/newhouse.html',
+}
+
+/**
+ * Build the datacenter URL for one report.
+ *
+ * @param {string} reportName - report id.
+ * @param {{ pageSize?: number, pageNumber?: number }} [options] - paging.
+ * @returns {string} request URL.
+ */
+export function buildUrl(reportName, { pageSize = 200, pageNumber = 1, city } = {}) {
+  // A per-city report needs the filter or it returns every city interleaved,
+  // which would look like one series whose value jumps between cities.
+  const filter = typeof city === 'string' && city !== '' ? `&filter=${encodeURIComponent(`(CITY="${city}")`)}` : ''
+  return (
+    'https://datacenter-web.eastmoney.com/api/data/v1/get' +
+    `?reportName=${encodeURIComponent(reportName)}` +
+    '&columns=ALL' +
+    `&pageSize=${pageSize}&pageNumber=${pageNumber}` +
+    '&sortColumns=REPORT_DATE&sortTypes=-1' +
+    '&source=WEB&client=WEB' +
+    filter
+  )
+}
+
+/**
+ * Landing page plus the data API URL.
+ *
+ * @param {string} seriesRef - report name.
+ * @param {{ column?: string, city?: string }} [params] - catalog params (column and city are part of the identity).
+ * @returns {{ adapterId: string, url: string, apiUrl: string, seriesRef: string, label: string }} source reference.
+ */
+export function sourceRef(seriesRef, params = {}) {
+  const identity = [params.column, params.city].filter((part) => typeof part === 'string' && part !== '').join(':')
+  return {
+    adapterId: id,
+    url: REPORT_PAGES[seriesRef] ?? 'https://data.eastmoney.com/cjsj/',
+    apiUrl: buildUrl(seriesRef, { city: params.city }),
+    seriesRef: identity === '' ? seriesRef : `${seriesRef}:${identity}`,
+    label,
+  }
+}
+
+/**
+ * Parse a datacenter payload into ascending points for one column.
+ *
+ * @param {string} body - response body.
+ * @param {string} seriesRef - report name (for errors).
+ * @param {string} column - column to read as 'v'.
+ * @returns {{ name: string, points: Array<{ t: string, v: number }> }} parsed series.
+ */
+export function parseReport(body, seriesRef, column) {
+  if (!column) {
+    throw new SourceError({
+      kind: 'parse',
+      adapterId: id,
+      seriesRef,
+      detail: 'params.column is required: one report serves several indicators',
+    })
+  }
+  const payload = parseJson(body, id, seriesRef)
+  if (payload === null || typeof payload !== 'object') {
+    throw new SourceError({ kind: 'parse', adapterId: id, seriesRef, detail: 'response is not a JSON object' })
+  }
+  if (payload.success === false) {
+    // Business failure on HTTP 200: the report does not exist (code 9501) or the
+    // request is malformed. Never retryable.
+    throw new SourceError({
+      kind: 'unsupported',
+      adapterId: id,
+      seriesRef,
+      detail: `datacenter rejected report ${seriesRef}: ${payload.message ?? 'unknown reason'} (code ${payload.code ?? '?'})`,
+    })
+  }
+  const rows = payload.result?.data
+  if (!Array.isArray(rows)) {
+    throw new SourceError({
+      kind: 'parse',
+      adapterId: id,
+      seriesRef,
+      detail: 'result.data is missing or not an array',
+    })
+  }
+  if (rows.length === 0) {
+    throw new SourceError({ kind: 'empty', adapterId: id, seriesRef, detail: 'the report returned zero rows' })
+  }
+
+  const points = []
+  let sawColumn = false
+  for (const row of rows) {
+    if (row === null || typeof row !== 'object') continue
+    const rawDate = row.REPORT_DATE
+    if (typeof rawDate !== 'string' || rawDate.length < 10) {
+      throw new SourceError({
+        kind: 'parse',
+        adapterId: id,
+        seriesRef,
+        detail: `row is missing a usable REPORT_DATE (${JSON.stringify(rawDate)})`,
+      })
+    }
+    if (column in row) sawColumn = true
+    const value = parseNumber(row[column])
+    if (value === undefined) continue
+    points.push({ t: rawDate.slice(0, 10), v: value })
+  }
+  if (!sawColumn) {
+    // A silent `undefined` per row is exactly the failure mode docs/04 §3 warns
+    // about: fail loudly instead of publishing an empty series.
+    throw new SourceError({
+      kind: 'parse',
+      adapterId: id,
+      seriesRef,
+      detail: `column ${column} does not exist in report ${seriesRef}; available: ${Object.keys(rows[0]).slice(0, 12).join(', ')}`,
+    })
+  }
+  if (points.length === 0) {
+    throw new SourceError({
+      kind: 'empty',
+      adapterId: id,
+      seriesRef,
+      detail: `column ${column} exists but every row was empty or unparseable`,
+    })
+  }
+  // The upstream sorts descending; statistics require ascending.
+  points.sort((a, b) => (a.t < b.t ? -1 : a.t > b.t ? 1 : 0))
+  return { name: `${seriesRef}:${column}`, points }
+}
+
+/**
+ * Fetch and normalize one macro series.
+ *
+ * @param {{ seriesRef: string, params?: { column?: string, city?: string }, range?: object, pageSize?: number }} req - request.
+ * @param {{ fetch: Function, clock: { now: () => Date }, signal?: AbortSignal }} deps - injected dependencies.
+ * @returns {Promise<object>} normalized 'RawSeries'.
+ */
+export async function fetchSeries(req, deps) {
+  const { seriesRef, params } = req
+  const column = params?.column
+  const city = params?.city
+  const url = buildUrl(seriesRef, { pageSize: req.pageSize ?? 200, city })
+  const { status, body } = await getText({
+    fetch: deps.fetch,
+    url,
+    adapterId: id,
+    seriesRef,
+    signal: deps.signal,
+  })
+  const { name, points } = parseReport(body, seriesRef, column)
+  return {
+    adapterId: id,
+    seriesRef,
+    points,
+    meta: { name, unit: undefined, freq: 'monthly' },
+    fetchedAt: deps.clock.now().toISOString(),
+    sourceRef: sourceRef(seriesRef, { column, city }),
+    raw: { url, httpStatus: status, rows: points.length, column, city },
+  }
+}

package/lib/sources/eastmoney-quote.js

@@ -0,0 +1,201 @@
+/**
+ * 东方财富行情 adapter — indices and bond indices from 'push2his' (docs/04 §2).
+ *
+ * The response is a JSON envelope whose 'data.klines' is a list of comma-joined
+ * CSV rows whose column order is exactly the order of the requested 'fields2'.
+ * We always request 'f51'(date) … 'f53'(close), which is why the close is the
+ * **third** column — the trap this adapter exists to contain.
+ *
+ * @module sources/eastmoney-quote
+ */
+import { SourceError } from '../core/types.js'
+import { addMonths, daysBetween } from '../core/time/range.js'
+import { parseNumber } from '../core/stats/series.js'
+import { withBar } from './ohlc.js'
+import { getText, parseJson } from './http.js'
+
+/** @type {string} */
+export const id = 'eastmoney-quote'
+
+/** @type {string} */
+export const label = '东方财富'
+
+/** @type {Array<{ kinds: string[], frequencies: string[] }>} */
+export const capabilities = [{ kinds: ['equity', 'bond'], frequencies: ['daily'] }]
+
+/** Its endpoint takes 'klt', so day/week/month bars are all real. */
+export const barSizes = true
+
+/** The field list this adapter requests, in order. */
+export const FIELDS2 = ['f51', 'f52', 'f53', 'f54', 'f55', 'f56', 'f57', 'f58', 'f59', 'f60', 'f61']
+
+/** Index of the close price inside the requested field list ('f53'). */
+export const CLOSE_FIELD_INDEX = FIELDS2.indexOf('f53')
+
+/**
+ * Column indices of one OHLC bar inside the requested field list.
+ *
+ * 'fields2' already asks for f51..f55, so open/high/low arrive with every kline
+ * row; the adapter used to read only the close and drop the rest.
+ */
+export const OHLC_FIELD_INDEX = {
+  date: FIELDS2.indexOf('f51'),
+  open: FIELDS2.indexOf('f52'),
+  close: FIELDS2.indexOf('f53'),
+  high: FIELDS2.indexOf('f54'),
+  low: FIELDS2.indexOf('f55'),
+}
+
+/** Default bars per request; enough for a 5-year window without a slow response. */
+const DEFAULT_LIMIT = 1600
+
+/**
+ * Build the kline URL for one secid.
+ *
+ * @param {string} seriesRef - 'market.code' secid, e.g. '1.000001'.
+ * @param {{ from: string, to: string }} range - requested window.
+ * @param {{ klt?: number, fqt?: number, lmt?: number }} [params] - catalog params.
+ * @returns {string} request URL.
+ */
+export function buildUrl(seriesRef, range, params = {}) {
+  const klt = params.klt ?? 101
+  const fqt = params.fqt ?? 1
+  const lmt = params.lmt ?? DEFAULT_LIMIT
+  return (
+    'https://push2his.eastmoney.com/api/qt/stock/kline/get' +
+    `?secid=${encodeURIComponent(seriesRef)}` +
+    '&fields1=f1,f2,f3,f4,f5,f6' +
+    `&fields2=${FIELDS2.join(',')}` +
+    `&klt=${klt}&fqt=${fqt}&end=20500101&lmt=${lmt}`
+  )
+}
+
+/**
+ * Human landing page for an index secid, plus the kline API URL.
+ *
+ * @param {string} seriesRef - secid.
+ * @returns {{ adapterId: string, url: string, apiUrl: string, seriesRef: string, label: string }} source reference.
+ */
+export function sourceRef(seriesRef) {
+  const [, code = seriesRef] = seriesRef.split('.')
+  const isDomestic = seriesRef.startsWith('1.') || seriesRef.startsWith('0.')
+  return {
+    adapterId: id,
+    url: isDomestic
+      ? `https://quote.eastmoney.com/zs${code}.html`
+      : 'https://quote.eastmoney.com/center/gridlist.html#global_globalindex',
+    apiUrl: buildUrl(seriesRef, { from: '2000-01-01', to: '2050-01-01' }),
+    seriesRef,
+    label,
+  }
+}
+
+/**
+ * Parse a kline payload into ascending daily bars (`v` is the close).
+ *
+ * @param {string} body - response body.
+ * @param {string} seriesRef - secid, echoed into errors.
+ * @returns {{ name: string, points: Array<{ t: string, v: number, o?: number, h?: number, l?: number }> }} parsed series.
+ */
+export function parseKlines(body, seriesRef) {
+  const payload = parseJson(body, id, seriesRef)
+  if (payload === null || typeof payload !== 'object') {
+    throw new SourceError({ kind: 'parse', adapterId: id, seriesRef, detail: 'response is not a JSON object' })
+  }
+  if (payload.data == null) {
+    // An envelope without `data` is either a business rejection (`rc`) or a body
+    // that is not a kline response at all. Only the former is "unsupported".
+    const rejected = payload.rc !== undefined && payload.rc !== 0
+    throw new SourceError({
+      kind: rejected ? 'unsupported' : 'parse',
+      adapterId: id,
+      seriesRef,
+      detail: rejected
+        ? `upstream rejected the secid (rc=${payload.rc})`
+        : 'response has no data.klines (not a kline payload)',
+    })
+  }
+  const rows = payload.data.klines
+  if (!Array.isArray(rows)) {
+    throw new SourceError({ kind: 'parse', adapterId: id, seriesRef, detail: 'data.klines is not an array' })
+  }
+  const points = []
+  for (const row of rows) {
+    const columns = String(row).split(',')
+    if (columns.length <= CLOSE_FIELD_INDEX) {
+      throw new SourceError({
+        kind: 'parse',
+        adapterId: id,
+        seriesRef,
+        detail: `kline row has ${columns.length} columns; expected at least ${CLOSE_FIELD_INDEX + 1} for the requested fields2`,
+      })
+    }
+    const value = parseNumber(columns[CLOSE_FIELD_INDEX])
+    if (value === undefined) continue
+    // The close is the observation; the rest of the bar rides along for candles,
+    // unless the four columns contradict each other.
+    points.push(withBar(
+      { t: columns[0], v: value },
+      { o: parseNumber(columns[OHLC_FIELD_INDEX.open]), h: parseNumber(columns[OHLC_FIELD_INDEX.high]), l: parseNumber(columns[OHLC_FIELD_INDEX.low]) },
+    ))
+  }
+  if (points.length === 0) {
+    throw new SourceError({ kind: 'empty', adapterId: id, seriesRef, detail: 'no usable kline rows in the response' })
+  }
+  points.sort((a, b) => (a.t < b.t ? -1 : a.t > b.t ? 1 : 0))
+  return { name: String(payload.data.name ?? seriesRef), points }
+}
+
+/**
+ * Fetch and normalize one index series.
+ *
+ * @param {{ seriesRef: string, range: { from: string, to: string }, params?: object }} req - request.
+ * @param {{ fetch: Function, clock: { now: () => Date }, signal?: AbortSignal }} deps - injected dependencies.
+ * @returns {Promise<object>} normalized 'RawSeries'.
+ */
+export async function fetchSeries(req, deps) {
+  const { seriesRef, range, params } = req
+  const url = buildUrl(seriesRef, range, params ?? {})
+  const { status, body } = await getText({
+    fetch: deps.fetch,
+    url,
+    adapterId: id,
+    seriesRef,
+    signal: deps.signal,
+  })
+  const { name, points } = parseKlines(body, seriesRef)
+  return {
+    adapterId: id,
+    seriesRef,
+    points,
+    meta: { name, unit: '点', freq: 'daily' },
+    fetchedAt: deps.clock.now().toISOString(),
+    sourceRef: sourceRef(seriesRef),
+    raw: { url, httpStatus: status, bars: points.length },
+  }
+}
+
+/**
+ * Mean gap between bars, used by tests to prove daily cadence without pulling
+ * in a trading calendar (docs/03 §2 rule 12).
+ *
+ * @param {Array<{ t: string, v: number }>} points - ascending points.
+ * @returns {number} mean gap in days.
+ */
+export function meanGapDays(points) {
+  if (points.length < 2) return 0
+  let total = 0
+  for (let i = 1; i < points.length; i += 1) total += daysBetween(points[i - 1].t, points[i].t)
+  return total / (points.length - 1)
+}
+
+/**
+ * The earliest date a 'lmt'-bar request can reach, for catalog documentation.
+ *
+ * @param {string} today - reference date.
+ * @param {number} [bars] - bar count.
+ * @returns {string} approximate earliest date.
+ */
+export function approximateHorizon(today, bars = DEFAULT_LIMIT) {
+  return addMonths(today, -Math.ceil((bars / 250) * 12))
+}

package/lib/sources/ecb.js

@@ -0,0 +1,179 @@
+/**
+ * ECB Data Portal (SDW) adapter (docs/04 §6).
+ *
+ * The CSV export of a dataflow is a long-format table with a wide header; the
+ * two columns that matter are 'TIME_PERIOD' and 'OBS_VALUE'. 'lastNObservations'
+ * must always be sent, or the response is the whole history.
+ *
+ * @module sources/ecb
+ */
+import { SourceError } from '../core/types.js'
+import { parseNumber } from '../core/stats/series.js'
+import { getText } from './http.js'
+
+/** @type {string} */
+export const id = 'ecb'
+
+/** @type {string} */
+export const label = 'ECB'
+
+/** @type {Array<{ kinds: string[], frequencies: string[] }>} */
+export const capabilities = [{ kinds: ['rates', 'fx'], frequencies: ['daily'] }]
+
+/** Default observation count; ~6 years of daily data. */
+const DEFAULT_OBSERVATIONS = 1600
+
+/**
+ * Build a CSV export URL for one dataflow key.
+ *
+ * @param {string} seriesRef - dataflow key, e.g. 'EXR/D.USD.EUR.SP00.A'.
+ * @param {{ lastNObservations?: number }} [params] - catalog params.
+ * @returns {string} request URL.
+ */
+export function buildUrl(seriesRef, { lastNObservations = DEFAULT_OBSERVATIONS } = {}) {
+  return (
+    `https://data-api.ecb.europa.eu/service/data/${seriesRef}` +
+    `?format=csvdata&lastNObservations=${lastNObservations}`
+  )
+}
+
+/**
+ * Dataset landing page plus the CSV API URL.
+ *
+ * @param {string} seriesRef - dataflow key.
+ * @returns {{ adapterId: string, url: string, apiUrl: string, seriesRef: string, label: string }} source reference.
+ */
+export function sourceRef(seriesRef) {
+  const flow = seriesRef.split('/')[0]
+  return {
+    adapterId: id,
+    url: `https://data.ecb.europa.eu/data/datasets/${flow}`,
+    apiUrl: buildUrl(seriesRef),
+    seriesRef,
+    label,
+  }
+}
+
+/**
+ * Parse an ECB csvdata export.
+ *
+ * @param {string} body - CSV text.
+ * @param {string} seriesRef - dataflow key (for errors).
+ * @returns {{ name: string, points: Array<{ t: string, v: number }> }} parsed series.
+ */
+export function parseCsvData(body, seriesRef) {
+  const lines = body.trim().split(/\r?\n/).filter((line) => line.trim() !== '')
+  if (lines.length === 0) {
+    throw new SourceError({ kind: 'empty', adapterId: id, seriesRef, detail: 'the CSV body is empty' })
+  }
+  const header = splitCsvLine(lines[0])
+  const timeIndex = header.indexOf('TIME_PERIOD')
+  const valueIndex = header.indexOf('OBS_VALUE')
+  if (timeIndex === -1 || valueIndex === -1) {
+    throw new SourceError({
+      kind: 'parse',
+      adapterId: id,
+      seriesRef,
+      detail: `CSV header lacks TIME_PERIOD/OBS_VALUE (got ${header.slice(0, 8).join(',')}…)`,
+    })
+  }
+  const points = []
+  for (const line of lines.slice(1)) {
+    const columns = splitCsvLine(line)
+    const period = columns[timeIndex]
+    const value = parseNumber(columns[valueIndex])
+    if (typeof period !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(period)) continue
+    if (value === undefined) continue // empty OBS_VALUE is a missing observation
+    points.push({ t: period, v: value })
+  }
+  if (points.length === 0) {
+    throw new SourceError({
+      kind: 'empty',
+      adapterId: id,
+      seriesRef,
+      detail: 'no observations with both a date and a value',
+    })
+  }
+  points.sort((a, b) => (a.t < b.t ? -1 : a.t > b.t ? 1 : 0))
+  const title = header[header.length - 1]
+  return { name: title && title.length > 3 ? title : seriesRef, points }
+}
+
+/**
+ * Split one CSV line, honouring double-quoted fields (the ECB title column
+ * contains commas).
+ *
+ * @param {string} line - CSV line.
+ * @returns {string[]} fields.
+ */
+export function splitCsvLine(line) {
+  const fields = []
+  let current = ''
+  let quoted = false
+  for (let i = 0; i < line.length; i += 1) {
+    const char = line[i]
+    if (quoted) {
+      if (char === '"') {
+        if (line[i + 1] === '"') {
+          current += '"'
+          i += 1
+        } else quoted = false
+      } else current += char
+    } else if (char === '"') {
+      quoted = true
+    } else if (char === ',') {
+      fields.push(current)
+      current = ''
+    } else current += char
+  }
+  fields.push(current)
+  return fields
+}
+
+/**
+ * Fetch and normalize one ECB series.
+ *
+ * @param {{ seriesRef: string, params?: { lastNObservations?: number } }} req - request.
+ * @param {{ fetch: Function, clock: { now: () => Date }, signal?: AbortSignal }} deps - injected dependencies.
+ * @returns {Promise<object>} normalized 'RawSeries'.
+ */
+export async function fetchSeries(req, deps) {
+  const { seriesRef, params } = req
+  const url = buildUrl(seriesRef, params ?? {})
+  const { status, body } = await getText({
+    fetch: deps.fetch,
+    url,
+    adapterId: id,
+    seriesRef,
+    signal: deps.signal,
+    allowHttpError: true,
+  })
+  if (status >= 400 && status < 500) {
+    throw new SourceError({
+      kind: 'unsupported',
+      adapterId: id,
+      seriesRef,
+      detail: `ECB data portal returned HTTP ${status} for ${seriesRef}; the dataflow key is not valid`,
+      httpStatus: status,
+    })
+  }
+  if (status >= 500) {
+    throw new SourceError({
+      kind: 'http',
+      adapterId: id,
+      seriesRef,
+      detail: `ECB data portal is unavailable (HTTP ${status})`,
+      httpStatus: status,
+    })
+  }
+  const { name, points } = parseCsvData(body, seriesRef)
+  return {
+    adapterId: id,
+    seriesRef,
+    points,
+    meta: { name, unit: undefined, freq: 'daily' },
+    fetchedAt: deps.clock.now().toISOString(),
+    sourceRef: sourceRef(seriesRef),
+    raw: { url, httpStatus: status, rows: points.length },
+  }
+}