@weaxs/dsh-stock-analysis 0.2.12 → 0.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,6 @@
2
2
  """Resolve Chinese stock names / pinyin / fuzzy input to stock codes."""
3
3
 
4
4
  import argparse
5
- import functools
6
5
  import json
7
6
  import re
8
7
  import sys
@@ -11,67 +10,33 @@ from pathlib import Path
11
10
 
12
11
  sys.path.insert(0, str(Path(__file__).resolve().parent))
13
12
 
14
- from _cache import DAY, cached # noqa: E402
15
13
  from _subproc import error_message, utf8_stdio # noqa: E402
16
- from stock_data import _akshare_retry, normalize_us_symbol # noqa: E402
17
-
18
-
19
- def _load_stock_map() -> list[dict]:
20
- def fetch():
21
- import akshare as ak
22
-
23
- df = _akshare_retry(ak.stock_zh_a_spot_em)
24
- records = []
25
- for row in df.to_dict("records"):
26
- code, name = str(row["代码"]), row["名称"]
27
- if (
28
- len(code) != 6
29
- or not code.isascii()
30
- or not code.isdigit()
31
- or not isinstance(name, str)
32
- or not name.strip()
33
- ):
34
- raise ValueError("Stock name map contains invalid codes or names")
35
- records.append({"code": code, "name": name})
36
- return records
37
-
38
- try:
39
- return cached("stock-name-map:v1", DAY, fetch)
40
- except Exception as e:
41
- raise RuntimeError(f"Failed to load stock list: {e}") from e
42
-
43
-
44
- @functools.cache # one attempt per process: import_parser resolves many names and failures are not disk-cached
45
- def _load_international_stock_map() -> list[dict]:
46
- # Odd provider rows are skipped: one unexpected code must not drop a whole market's names.
47
- def fetch_hk():
48
- import akshare as ak
49
-
50
- records = []
51
- for row in _akshare_retry(ak.stock_hk_spot_em).to_dict("records"):
52
- code, name = str(row["代码"]).split(".", 1)[0].zfill(5), row["名称"]
53
- if code.isdigit() and len(code) == 5 and isinstance(name, str) and name.strip():
54
- records.append({"code": f"{code}.HK", "name": name})
55
- return records
56
-
57
- def fetch_us():
58
- import akshare as ak
59
-
60
- records = []
61
- for row in _akshare_retry(ak.stock_us_spot_em).to_dict("records"):
62
- code, name = normalize_us_symbol(row["代码"]), row["名称"]
63
- if re.fullmatch(r"[A-Z0-9-]{1,10}", code) and isinstance(name, str) and name.strip():
64
- records.append({"code": code, "name": name})
65
- return records
66
-
67
- records = []
68
- for key, fetch in (("stock-name-map:hk:v1", fetch_hk), ("stock-name-map:us:v1", fetch_us)):
69
- try:
70
- records.extend(cached(key, DAY, fetch))
71
- except Exception:
72
- # One unavailable market must not hide the other markets' names.
14
+ from stock_data import _em_codetable, normalize_us_symbol # noqa: E402
15
+
16
+ _A_TYPES = {"沪A", "深A", "京A", "科创板", "创业板"}
17
+
18
+
19
+ def _search(query: str, count: int) -> list[dict]:
20
+ """Eastmoney code-table search (code / name / pinyin / initials), kept to A/HK/US listed stocks."""
21
+ stocks = []
22
+ for row in _em_codetable(query, count):
23
+ code, name, market, kind = (
24
+ str(row.get("code", "")),
25
+ row.get("shortName"),
26
+ row.get("market"),
27
+ row.get("securityTypeName"),
28
+ )
29
+ if not isinstance(name, str) or not name.strip():
73
30
  continue
74
- return records
31
+ if market in (0, 1) and kind in _A_TYPES and re.fullmatch(r"\d{6}", code):
32
+ stocks.append({"code": code, "name": name})
33
+ elif market == 116 and kind == "港股" and re.fullmatch(r"\d{5}", code):
34
+ stocks.append({"code": f"{code}.HK", "name": name})
35
+ elif market in (105, 106, 107) and kind == "美股":
36
+ symbol = normalize_us_symbol(code)
37
+ if re.fullmatch(r"[A-Z0-9-]{1,10}", symbol):
38
+ stocks.append({"code": symbol, "name": name})
39
+ return stocks
75
40
 
76
41
 
77
42
  def _to_pinyin(text: str) -> str:
@@ -140,27 +105,15 @@ def _resolve_stocks(query: str, top: int, stocks: list[dict]) -> list[dict]:
140
105
 
141
106
 
142
107
  def resolve(query: str, top: int = 5) -> list[dict]:
143
- load_error = None
144
- try:
145
- stocks = _load_stock_map()
146
- local = _resolve_stocks(query, top, stocks)
147
- except Exception as exc:
148
- stocks, local, load_error = [], [], exc
149
108
  query = query.strip()
150
- code_like = re.fullmatch(r"[A-Za-z][A-Za-z0-9.-]{0,14}", query)
151
- # A substring hit (京东 → 京东方A) is not proof the user meant an A share (京东集团-SW is HK).
152
- partial_name = bool(local) and "score" not in local[0] and query != local[0]["name"] and query in local[0]["name"]
153
- # Pinyin/initials scores are not proof either (特斯拉 → 天士力 scores 0.9), so only literal hits short-circuit.
154
- if local and "score" not in local[0] and not code_like and not partial_name:
155
- return local
156
- international = _load_international_stock_map()
157
- if international:
158
- hits = _resolve_stocks(query, top, stocks + international)
159
- if load_error is None or (hits and "score" not in hits[0]): # fuzzy-only hits are a miss
160
- return hits
161
- if load_error is not None:
162
- raise load_error
163
- return local
109
+ if not query:
110
+ return []
111
+ try:
112
+ # The provider ranks loosely; over-fetch so the local exact/prefix/contains/pinyin order decides.
113
+ stocks = _search(query, max(top, 20))
114
+ except Exception as e:
115
+ raise RuntimeError(f"Failed to search stock names: {error_message(e)}") from e
116
+ return _resolve_stocks(query, top, stocks)
164
117
 
165
118
 
166
119
  def main():
package/tools/screener.py CHANGED
@@ -21,7 +21,6 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
21
21
 
22
22
  from _subproc import command_timeout, run_tool, utf8_stdio
23
23
  from market_review import calc_temperature
24
- from stock_data import compute_market_stats
25
24
 
26
25
  DEFAULT_FILTERS = {
27
26
  "pe_min": 0,
@@ -56,12 +55,10 @@ def _fetch_json(args: list[str], timeout: int | None = None) -> dict | list | No
56
55
  return run_tool("stock_data.py", args, timeout=timeout, preserve_errors=True)
57
56
 
58
57
 
59
- def fetch_snapshot(market: str) -> list:
60
- # a cold full-market snapshot can exceed 30s (CI runners) — allow longer; with
61
- # all eastmoney/sina legs timing out, the tencent fallback leg needs ~2min more
62
- return _fetch_json(
63
- ["market_snapshot", "--market", market], timeout=command_timeout("stock_data.py", ["market_snapshot"])
64
- )
58
+ def fetch_snapshot(market: str, filters: dict) -> dict:
59
+ """Stocks at or above filters["market_cap_min"], largest first (A also filters server-side)."""
60
+ args = ["market_snapshot", "--market", market, "--filters", json.dumps(filters)]
61
+ return _fetch_json(args, timeout=command_timeout("stock_data.py", ["market_snapshot"]))
65
62
 
66
63
 
67
64
  def load_config(path: str) -> dict:
@@ -167,19 +164,18 @@ def sentiment_multiplier(score: float) -> float:
167
164
  return round(0.8 + score / 100 * 0.4, 3) # linear map to [0.8, 1.2]
168
165
 
169
166
 
170
- def compute_sentiment(market: str, snapshot: list | None = None) -> dict:
167
+ def compute_sentiment(market: str) -> dict:
171
168
  """Market sentiment gate; degrades to a neutral multiplier of 1.0 on any failure."""
172
169
  fallback = {"score": None, "level": "unknown", "signal": None, "multiplier": 1.0}
173
170
  region = _INDEX_REGION.get(market)
174
171
  if region is None:
175
172
  return fallback
176
173
  try:
177
- stats = None
178
- if isinstance(snapshot, list) and snapshot:
179
- try:
180
- stats = compute_market_stats(snapshot)
181
- except Exception:
182
- stats = None
174
+ stats = _fetch_json(
175
+ ["market_stats", "--market", market], timeout=command_timeout("stock_data.py", ["market_stats"])
176
+ )
177
+ if not isinstance(stats, dict) or "error" in stats:
178
+ stats = None
183
179
  indices = _fetch_json(["market_indices", "--region", region])
184
180
  if not isinstance(indices, list) or not indices:
185
181
  return fallback
@@ -372,22 +368,29 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
372
368
  )
373
369
  if dividend_requested and market not in ("A", "HK"):
374
370
  return {"error": "dividend screening currently covers A/HK only", "market": market}
375
- raw = fetch_snapshot(market)
376
- if isinstance(raw, dict) and "error" in raw:
377
- return raw
378
- # snapshot legs wrap a total failure as a one-row [{"error": ...}] list — surface it
379
- # instead of letting the error row walk the scoring pipeline as a ghost candidate
380
- if isinstance(raw, list) and len(raw) == 1 and isinstance(raw[0], dict) and "error" in raw[0]:
381
- return {**raw[0], "market": market}
382
- if raw is None:
371
+ # Sentiment is an independent market_stats call; overlap it with the snapshot fetch.
372
+ pool = ThreadPoolExecutor(max_workers=1)
373
+ sentiment_future = pool.submit(compute_sentiment, market)
374
+ pool.shutdown(wait=False)
375
+ snapshot = fetch_snapshot(market, filters)
376
+ if isinstance(snapshot, dict) and "error" in snapshot:
377
+ return snapshot
378
+ if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list):
383
379
  return {"error": "failed to fetch market snapshot", "market": market}
384
380
 
385
381
  l2_enabled = bool(l2 or (config or {}).get("l2"))
386
- sentiment = compute_sentiment(market, raw if isinstance(raw, list) else None)
382
+ sentiment = sentiment_future.result()
387
383
  multiplier = sentiment["multiplier"]
388
384
 
389
- df = pd.DataFrame(raw)
390
- total = len(df)
385
+ # An empty server-filtered page still reports every requested filter as applied.
386
+ columns = ["symbol", *{key.rsplit("_", 1)[0] for key in filters}]
387
+ df = pd.DataFrame(snapshot["rows"]) if snapshot["rows"] else pd.DataFrame(columns=columns)
388
+ total = snapshot.get("total_stocks")
389
+ scan_meta = {}
390
+ if snapshot.get("truncated"):
391
+ scan_meta["warnings"] = [
392
+ f"scanned only the {len(df)} largest stocks by market cap; raise market_cap_min to cover the rest"
393
+ ]
391
394
 
392
395
  for c in ["price", "change_pct", "volume", "turnover", "pe", "pb", "market_cap", "turnover_rate", "volume_ratio"]:
393
396
  if c in df.columns:
@@ -403,7 +406,11 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
403
406
  try:
404
407
  from value_data import dividend_ranking
405
408
 
406
- ranked = dividend_ranking(report_date=(config or {}).get("dividend_report_date", ""), market=market)
409
+ ranked = dividend_ranking(
410
+ report_date=(config or {}).get("dividend_report_date", ""),
411
+ market=market,
412
+ codes=list(df["symbol"]) if market == "A" else None,
413
+ )
407
414
 
408
415
  def dividend_key(symbol):
409
416
  return symbol.split(".")[0].lstrip("0") if market == "HK" else symbol
@@ -441,6 +448,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
441
448
  "market_sentiment": sentiment,
442
449
  "l2_enabled": l2_enabled,
443
450
  **dividend_meta,
451
+ **scan_meta,
444
452
  }
445
453
 
446
454
  df = compute_scores(df)
@@ -531,6 +539,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
531
539
  "market_sentiment": sentiment,
532
540
  "l2_enabled": l2_enabled,
533
541
  **dividend_meta,
542
+ **scan_meta,
534
543
  }
535
544
 
536
545
 
@@ -19,6 +19,7 @@ from _cache import DAY, WEEK, cached
19
19
  from _subproc import utf8_stdio
20
20
 
21
21
  _last_request = 0.0
22
+ SEC_MIN_INTERVAL = 0.5
22
23
 
23
24
 
24
25
  def identity():
@@ -31,7 +32,7 @@ def identity():
31
32
  def sec_get(url, as_json=True, stream=False):
32
33
  global _last_request
33
34
  agent = identity()
34
- time.sleep(max(0, 0.5 - (time.monotonic() - _last_request)))
35
+ time.sleep(max(0, SEC_MIN_INTERVAL - (time.monotonic() - _last_request)))
35
36
  _last_request = time.monotonic()
36
37
  try:
37
38
  response = requests.get(
@@ -500,7 +500,7 @@ def portfolio(p):
500
500
  centered = raw_vector - raw_vector.mean()
501
501
  history_data[id(history)] = {
502
502
  "days": days,
503
- "days_array": np.array(days, dtype=str),
503
+ "ordinals": np.array(days, dtype="datetime64[D]").astype(np.int64),
504
504
  "center": center,
505
505
  "scale": scale,
506
506
  "raw": raw_vector,
@@ -508,13 +508,14 @@ def portfolio(p):
508
508
  "norm": np.linalg.norm(centered),
509
509
  }
510
510
 
511
- def scaled_vector(history, common):
511
+ def scaled_vector(history, indices):
512
512
  data = history_data[id(history)]
513
- if common == data["days"]:
513
+ days = data["days"]
514
+ if len(indices) == len(days):
514
515
  return data["centered"], data["norm"]
515
516
 
516
517
  def exact_common_vector():
517
- values = history_returns(history, common, returns_cache)
518
+ values = history_returns(history, [days[index] for index in indices], returns_cache)
518
519
  center = values[0]
519
520
  deviations = [value - center for value in values]
520
521
  scale = max((abs(value) for value in deviations), default=0)
@@ -525,10 +526,9 @@ def portfolio(p):
525
526
  center, scale = data["center"], data["scale"]
526
527
  if scale == 0 or scale > 1000000 or scale < Fraction(1, 1000000):
527
528
  return exact_common_vector()
528
- indices = np.searchsorted(data["days_array"], common)
529
529
  subset = data["raw"][indices[1:] - 1].copy()
530
530
  for index in np.flatnonzero(np.diff(indices) != 1):
531
- gap = history_returns(history, [common[index], common[index + 1]], returns_cache)[0]
531
+ gap = history_returns(history, [days[indices[index]], days[indices[index + 1]]], returns_cache)[0]
532
532
  try:
533
533
  gap_value = float((gap - center) / scale)
534
534
  except OverflowError:
@@ -550,25 +550,23 @@ def portfolio(p):
550
550
 
551
551
  def correlation_row(pair):
552
552
  a, b = pair
553
- common = [day for day in history_data[id(histories[a])]["days"] if day in histories[b]]
553
+ _, a_indices, b_indices = np.intersect1d(
554
+ history_data[id(histories[a])]["ordinals"],
555
+ history_data[id(histories[b])]["ordinals"],
556
+ assume_unique=True,
557
+ return_indices=True,
558
+ )
554
559
  corr = None
555
- if len(common) >= 4:
556
- x, x_norm = scaled_vector(histories[a], common)
557
- y, y_norm = scaled_vector(histories[b], common)
560
+ if len(a_indices) >= 4:
561
+ x, x_norm = scaled_vector(histories[a], a_indices)
562
+ y, y_norm = scaled_vector(histories[b], b_indices)
558
563
  if x_norm and y_norm:
559
564
  correlation = float(np.dot(x, y) / (x_norm * y_norm))
560
565
  corr = max(-1.0, min(1.0, correlation))
561
- return {"symbols": [a, b], "correlation": corr, "return_samples": max(len(common) - 1, 0)}
566
+ return {"symbols": [a, b], "correlation": corr, "return_samples": max(len(a_indices) - 1, 0)}
562
567
 
563
568
  pairs = [(a, b) for i, a in enumerate(symbols) for b in symbols[i + 1 :]]
564
- if pairs and max(len(history) for history in histories.values()) * len(pairs) > 200000:
565
- import os
566
- from concurrent.futures import ThreadPoolExecutor
567
-
568
- with ThreadPoolExecutor(max_workers=min(2, os.cpu_count() or 1)) as pool:
569
- correlations = list(pool.map(correlation_row, pairs))
570
- else:
571
- correlations = [correlation_row(pair) for pair in pairs]
569
+ correlations = [correlation_row(pair) for pair in pairs]
572
570
  warnings = [
573
571
  "Price P&L uses current FX, excludes historical FX P&L, fees and distributions; stop risk is a scenario, not a guaranteed loss limit.",
574
572
  "Correlations use local-price returns on common dates, exclude FX, and do not forecast future dependence.",