@weaxs/openclaw-stock-analysis 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +32 -28
- package/index.ts +32 -28
- package/package.json +1 -1
- package/skills/stock-analysis/SKILL.md +12 -1
- package/skills/stock-screener/SKILL.md +8 -2
- package/tools/market_review.py +2 -2
- package/tools/screener.py +236 -19
- package/tools/search_intel.py +129 -105
- package/tools/stock_data.py +76 -65
package/tools/screener.py
CHANGED
|
@@ -1,16 +1,28 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
"""AlphaSift L1 — multi-factor stock screener with hard filter + scoring.
|
|
2
|
+
"""AlphaSift L1 — multi-factor stock screener with hard filter + scoring.
|
|
3
|
+
|
|
4
|
+
L1: snapshot → hard filters → value/momentum/liquidity composite.
|
|
5
|
+
Regime gate: market sentiment temperature scales the final score (multiplier in [0.8, 1.2]).
|
|
6
|
+
L2 (opt-in via --l2 / config l2: true): per-candidate quality/growth/momentum/volatility/moneyflow factors.
|
|
7
|
+
"""
|
|
3
8
|
|
|
4
9
|
import argparse
|
|
5
10
|
import json
|
|
6
11
|
import os
|
|
7
12
|
import subprocess
|
|
8
13
|
import sys
|
|
14
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
15
|
+
from pathlib import Path
|
|
9
16
|
|
|
10
17
|
import numpy as np
|
|
11
18
|
import pandas as pd
|
|
12
19
|
import yaml
|
|
13
20
|
|
|
21
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
22
|
+
|
|
23
|
+
from market_review import calc_temperature
|
|
24
|
+
from stock_data import compute_market_stats
|
|
25
|
+
|
|
14
26
|
DEFAULT_FILTERS = {
|
|
15
27
|
"pe_min": 0,
|
|
16
28
|
"pe_max": 100,
|
|
@@ -24,16 +36,40 @@ DEFAULT_FILTERS = {
|
|
|
24
36
|
"volume_ratio_min": 0.5,
|
|
25
37
|
}
|
|
26
38
|
|
|
39
|
+
L2_WEIGHTS = {
|
|
40
|
+
"value": 0.15,
|
|
41
|
+
"quality": 0.20,
|
|
42
|
+
"growth": 0.15,
|
|
43
|
+
"momentum": 0.20,
|
|
44
|
+
"liquidity": 0.10,
|
|
45
|
+
"moneyflow": 0.15,
|
|
46
|
+
"low_volatility": 0.05,
|
|
47
|
+
}
|
|
27
48
|
|
|
28
|
-
|
|
49
|
+
_INDEX_REGION = {"A": "cn", "HK": "hk", "US": "us"}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _fetch_json(args: list[str], timeout: int = 30) -> dict | list | None:
|
|
53
|
+
"""Run a stock_data.py CLI command and parse its JSON stdout; None on any failure."""
|
|
29
54
|
script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "stock_data.py")
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
55
|
+
try:
|
|
56
|
+
r = subprocess.run(
|
|
57
|
+
[sys.executable, script, *args],
|
|
58
|
+
capture_output=True,
|
|
59
|
+
text=True,
|
|
60
|
+
encoding="utf-8",
|
|
61
|
+
timeout=timeout,
|
|
62
|
+
)
|
|
63
|
+
if r.returncode == 0 and r.stdout.strip():
|
|
64
|
+
return json.loads(r.stdout)
|
|
65
|
+
except Exception:
|
|
66
|
+
pass
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def fetch_snapshot(market: str) -> list:
|
|
71
|
+
# a cold full-market snapshot can exceed 30s (CI runners) — allow longer
|
|
72
|
+
return _fetch_json(["market_snapshot", "--market", market], timeout=120)
|
|
37
73
|
|
|
38
74
|
|
|
39
75
|
def load_config(path: str) -> dict:
|
|
@@ -111,10 +147,175 @@ def compute_scores(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
111
147
|
return df
|
|
112
148
|
|
|
113
149
|
|
|
114
|
-
def
|
|
150
|
+
def sentiment_multiplier(score: float) -> float:
|
|
151
|
+
return round(0.8 + score / 100 * 0.4, 3) # linear map to [0.8, 1.2]
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def compute_sentiment(market: str, snapshot: list | None = None) -> dict:
|
|
155
|
+
"""Market sentiment gate; degrades to a neutral multiplier of 1.0 on any failure."""
|
|
156
|
+
fallback = {"score": None, "level": "unknown", "signal": None, "multiplier": 1.0}
|
|
157
|
+
region = _INDEX_REGION.get(market)
|
|
158
|
+
if region is None:
|
|
159
|
+
return fallback
|
|
160
|
+
try:
|
|
161
|
+
stats = None
|
|
162
|
+
if isinstance(snapshot, list) and snapshot:
|
|
163
|
+
try:
|
|
164
|
+
stats = compute_market_stats(snapshot)
|
|
165
|
+
except Exception:
|
|
166
|
+
stats = None
|
|
167
|
+
indices = _fetch_json(["market_indices", "--region", region])
|
|
168
|
+
if not isinstance(indices, list) or not indices:
|
|
169
|
+
return fallback
|
|
170
|
+
temp = calc_temperature(stats, indices)
|
|
171
|
+
score = temp.get("score")
|
|
172
|
+
if not isinstance(score, (int, float)):
|
|
173
|
+
return fallback
|
|
174
|
+
return {
|
|
175
|
+
"score": score,
|
|
176
|
+
"level": temp.get("level", "unknown"),
|
|
177
|
+
"signal": temp.get("signal"),
|
|
178
|
+
"multiplier": sentiment_multiplier(float(score)),
|
|
179
|
+
}
|
|
180
|
+
except Exception:
|
|
181
|
+
return fallback
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _enrich_one(symbol: str, market: str) -> dict:
|
|
185
|
+
"""Fetch raw L2 indicators for one candidate; missing factor sources just yield missing keys."""
|
|
186
|
+
result: dict = {}
|
|
187
|
+
try:
|
|
188
|
+
kline = _fetch_json(["kline", symbol, "--period", "daily", "--count", "250"])
|
|
189
|
+
if isinstance(kline, list) and len(kline) >= 2:
|
|
190
|
+
closes = pd.Series([float(k["close"]) for k in kline if k.get("close") is not None])
|
|
191
|
+
highs = [float(k["high"]) for k in kline if k.get("high") is not None]
|
|
192
|
+
if len(closes) >= 21:
|
|
193
|
+
result["ret_20d"] = round(float(closes.iloc[-1] / closes.iloc[-21] - 1) * 100, 2)
|
|
194
|
+
if len(closes) >= 61:
|
|
195
|
+
result["ret_60d"] = round(float(closes.iloc[-1] / closes.iloc[-61] - 1) * 100, 2)
|
|
196
|
+
if len(closes) and highs:
|
|
197
|
+
peak = max(highs)
|
|
198
|
+
if peak > 0:
|
|
199
|
+
result["high_250_proximity"] = round(float(closes.iloc[-1]) / peak, 4)
|
|
200
|
+
rets = closes.pct_change().dropna().tail(20)
|
|
201
|
+
if len(rets) >= 2:
|
|
202
|
+
result["vol20"] = round(float(rets.std()) * float(np.sqrt(252)), 4)
|
|
203
|
+
except Exception:
|
|
204
|
+
pass
|
|
205
|
+
|
|
206
|
+
if market == "A":
|
|
207
|
+
fund = _fetch_json(["fundamental_context", symbol])
|
|
208
|
+
if isinstance(fund, dict) and "error" not in fund:
|
|
209
|
+
prof = fund.get("profitability") or {}
|
|
210
|
+
for key in ("roe", "gross_margin", "net_margin"):
|
|
211
|
+
if prof.get(key) is not None:
|
|
212
|
+
result[key] = prof[key]
|
|
213
|
+
growth = fund.get("growth") or {}
|
|
214
|
+
for key in ("revenue_yoy", "net_income_yoy"):
|
|
215
|
+
if growth.get(key) is not None:
|
|
216
|
+
result[key] = growth[key]
|
|
217
|
+
else:
|
|
218
|
+
fin = _fetch_json(["financials", symbol])
|
|
219
|
+
if isinstance(fin, dict) and "error" not in fin:
|
|
220
|
+
for key in ("roe", "profit_margin"):
|
|
221
|
+
if fin.get(key) is not None:
|
|
222
|
+
result[key] = fin[key]
|
|
223
|
+
|
|
224
|
+
if market == "A":
|
|
225
|
+
flow = _fetch_json(["capital_flow", symbol, "--mode", "summary"])
|
|
226
|
+
if isinstance(flow, dict) and "error" not in flow and flow.get("main_net_5d") is not None:
|
|
227
|
+
result["main_net_5d"] = flow["main_net_5d"]
|
|
228
|
+
return result
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _cross_sectional_scores(enrichments: dict) -> dict:
|
|
232
|
+
"""Min-max normalize enrichment factors across the candidate set; groups fully missing are skipped."""
|
|
233
|
+
fdf = pd.DataFrame.from_dict(enrichments, orient="index") if enrichments else pd.DataFrame()
|
|
234
|
+
out: dict = {sym: {} for sym in fdf.index}
|
|
235
|
+
|
|
236
|
+
def col(name: str) -> pd.Series | None:
|
|
237
|
+
return pd.to_numeric(fdf[name], errors="coerce") if name in fdf.columns else None
|
|
238
|
+
|
|
239
|
+
def add_group(group: str, parts: list):
|
|
240
|
+
valid = [p for p in parts if p is not None and p.notna().any()]
|
|
241
|
+
if not valid:
|
|
242
|
+
return
|
|
243
|
+
combined = pd.concat(valid, axis=1).mean(axis=1)
|
|
244
|
+
for sym, v in combined.items():
|
|
245
|
+
if not pd.isna(v):
|
|
246
|
+
out[sym][group] = float(v)
|
|
247
|
+
|
|
248
|
+
margin = col("gross_margin")
|
|
249
|
+
profit_margin = col("profit_margin")
|
|
250
|
+
if margin is None:
|
|
251
|
+
margin = profit_margin
|
|
252
|
+
elif profit_margin is not None:
|
|
253
|
+
margin = margin.combine_first(profit_margin)
|
|
254
|
+
add_group("quality", [_minmax(s) for s in (col("roe"), margin, col("net_margin")) if s is not None])
|
|
255
|
+
add_group(
|
|
256
|
+
"growth", [_minmax(s.clip(-50, 100)) for s in (col("revenue_yoy"), col("net_income_yoy")) if s is not None]
|
|
257
|
+
)
|
|
258
|
+
add_group(
|
|
259
|
+
"momentum",
|
|
260
|
+
[_minmax(s) for s in (col("ret_20d"), col("ret_60d"), col("high_250_proximity")) if s is not None],
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
flow = col("main_net_5d")
|
|
264
|
+
if flow is not None:
|
|
265
|
+
add_group("moneyflow", [_minmax(flow)])
|
|
266
|
+
|
|
267
|
+
vol = col("vol20")
|
|
268
|
+
if vol is not None:
|
|
269
|
+
add_group("low_volatility", [100 - _minmax(vol)])
|
|
270
|
+
|
|
271
|
+
return out
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _apply_l2(candidates: list, market: str, multiplier: float) -> None:
|
|
275
|
+
"""Enrich candidates with L2 factors and rewrite each scores['final'] as composite_l2 × multiplier."""
|
|
276
|
+
symbols = [c.get("symbol") for c in candidates]
|
|
277
|
+
enrichments: dict = {}
|
|
278
|
+
with ThreadPoolExecutor(max_workers=6) as executor:
|
|
279
|
+
futures = {executor.submit(_enrich_one, sym, market): sym for sym in symbols if sym}
|
|
280
|
+
for future, sym in futures.items():
|
|
281
|
+
try:
|
|
282
|
+
enrichments[sym] = future.result() or {}
|
|
283
|
+
except Exception:
|
|
284
|
+
enrichments[sym] = {}
|
|
285
|
+
|
|
286
|
+
factor_scores = _cross_sectional_scores(enrichments)
|
|
287
|
+
|
|
288
|
+
for c in candidates:
|
|
289
|
+
sym = c.get("symbol")
|
|
290
|
+
groups = factor_scores.get(sym, {})
|
|
291
|
+
c["enriched"] = bool(enrichments.get(sym))
|
|
292
|
+
scores = c["scores"]
|
|
293
|
+
components = {
|
|
294
|
+
"value": scores["value"],
|
|
295
|
+
"liquidity": scores["liquidity"],
|
|
296
|
+
}
|
|
297
|
+
# momentum is L2-only: candidates without kline factors drop the group and renormalize weights.
|
|
298
|
+
components.update(
|
|
299
|
+
{g: groups[g] for g in ("momentum", "quality", "growth", "moneyflow", "low_volatility") if g in groups}
|
|
300
|
+
)
|
|
301
|
+
scores.update(
|
|
302
|
+
{g: round(groups[g], 1) for g in ("quality", "growth", "moneyflow", "low_volatility") if g in groups}
|
|
303
|
+
)
|
|
304
|
+
total_weight = sum(L2_WEIGHTS[g] for g in components)
|
|
305
|
+
composite_l2 = sum(L2_WEIGHTS[g] * components[g] for g in components) / total_weight
|
|
306
|
+
scores["final"] = round(composite_l2 * multiplier, 1)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict:
|
|
115
310
|
raw = fetch_snapshot(market)
|
|
116
311
|
if isinstance(raw, dict) and "error" in raw:
|
|
117
312
|
return raw
|
|
313
|
+
if raw is None:
|
|
314
|
+
return {"error": "failed to fetch market snapshot", "market": market}
|
|
315
|
+
|
|
316
|
+
l2_enabled = bool(l2 or (config or {}).get("l2"))
|
|
317
|
+
sentiment = compute_sentiment(market, raw if isinstance(raw, list) else None)
|
|
318
|
+
multiplier = sentiment["multiplier"]
|
|
118
319
|
|
|
119
320
|
df = pd.DataFrame(raw)
|
|
120
321
|
total = len(df)
|
|
@@ -140,18 +341,21 @@ def screen(market: str, top: int, config: dict | None) -> dict:
|
|
|
140
341
|
"returned_count": 0,
|
|
141
342
|
"filters_applied": filters,
|
|
142
343
|
"candidates": [],
|
|
344
|
+
"market_sentiment": sentiment,
|
|
345
|
+
"l2_enabled": l2_enabled,
|
|
143
346
|
}
|
|
144
347
|
|
|
145
348
|
df = compute_scores(df)
|
|
146
349
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
if sort_by in df.columns:
|
|
152
|
-
df = df.sort_values(sort_by, ascending=ascending)
|
|
350
|
+
# L2 re-scores by final afterwards; config sort_by/sort_order only apply to the non-L2 path.
|
|
351
|
+
if l2_enabled or not config:
|
|
352
|
+
sort_by, ascending = "composite_score", False
|
|
153
353
|
else:
|
|
154
|
-
|
|
354
|
+
sort_by = config.get("sort_by", "composite_score")
|
|
355
|
+
ascending = config.get("sort_order", "desc") == "asc"
|
|
356
|
+
if sort_by not in df.columns:
|
|
357
|
+
sort_by, ascending = "composite_score", False
|
|
358
|
+
df = df.sort_values(sort_by, ascending=ascending)
|
|
155
359
|
|
|
156
360
|
df = df.head(top)
|
|
157
361
|
|
|
@@ -173,14 +377,22 @@ def screen(market: str, top: int, config: dict | None) -> dict:
|
|
|
173
377
|
v = row.get(field)
|
|
174
378
|
if v is not None and not (isinstance(v, float) and np.isnan(v)):
|
|
175
379
|
entry[field] = round(float(v), 2) if isinstance(v, (float, np.floating)) else v
|
|
380
|
+
composite = float(row.get("composite_score", 0))
|
|
176
381
|
entry["scores"] = {
|
|
177
382
|
"value": round(float(row.get("value_score", 0)), 1),
|
|
178
383
|
"momentum": round(float(row.get("momentum_score", 0)), 1),
|
|
179
384
|
"liquidity": round(float(row.get("liquidity_score", 0)), 1),
|
|
180
|
-
"composite": round(
|
|
385
|
+
"composite": round(composite, 1),
|
|
386
|
+
"final": round(composite * multiplier, 1),
|
|
181
387
|
}
|
|
182
388
|
candidates.append(entry)
|
|
183
389
|
|
|
390
|
+
if l2_enabled:
|
|
391
|
+
_apply_l2(candidates, market, multiplier)
|
|
392
|
+
candidates.sort(key=lambda c: c["scores"]["final"], reverse=True)
|
|
393
|
+
for rank, entry in enumerate(candidates, 1):
|
|
394
|
+
entry["rank"] = rank
|
|
395
|
+
|
|
184
396
|
return {
|
|
185
397
|
"market": market,
|
|
186
398
|
"total_stocks": total,
|
|
@@ -188,6 +400,8 @@ def screen(market: str, top: int, config: dict | None) -> dict:
|
|
|
188
400
|
"returned_count": len(candidates),
|
|
189
401
|
"filters_applied": filters,
|
|
190
402
|
"candidates": candidates,
|
|
403
|
+
"market_sentiment": sentiment,
|
|
404
|
+
"l2_enabled": l2_enabled,
|
|
191
405
|
}
|
|
192
406
|
|
|
193
407
|
|
|
@@ -199,6 +413,9 @@ def main():
|
|
|
199
413
|
p.add_argument("--market", default="A", choices=["A", "HK", "US"])
|
|
200
414
|
p.add_argument("--top", type=int, default=20)
|
|
201
415
|
p.add_argument("--config", help="Path to YAML config file")
|
|
416
|
+
p.add_argument(
|
|
417
|
+
"--l2", action="store_true", help="Enable L2 factor enrichment (quality/growth/momentum/volatility/moneyflow)"
|
|
418
|
+
)
|
|
202
419
|
|
|
203
420
|
args = parser.parse_args()
|
|
204
421
|
if not args.command:
|
|
@@ -206,7 +423,7 @@ def main():
|
|
|
206
423
|
sys.exit(1)
|
|
207
424
|
|
|
208
425
|
config = load_config(args.config) if args.config else None
|
|
209
|
-
result = screen(args.market, args.top, config)
|
|
426
|
+
result = screen(args.market, args.top, config, l2=args.l2)
|
|
210
427
|
print(json.dumps(result, ensure_ascii=False, default=str))
|
|
211
428
|
|
|
212
429
|
|
package/tools/search_intel.py
CHANGED
|
@@ -27,123 +27,130 @@ def _cache_set(key: str, data):
|
|
|
27
27
|
|
|
28
28
|
# --------------- Web Search ---------------
|
|
29
29
|
|
|
30
|
+
_SECRET_PARAM_RE = re.compile(r"(?i)((?:api[_-]?key|token|secret|access[_-]?token)=)[^&\s]+")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _scrub_secrets(msg: str) -> str:
|
|
34
|
+
"""Error text may embed request URLs (requests connection errors) — never leak keys to stdout JSON."""
|
|
35
|
+
return _SECRET_PARAM_RE.sub(r"\1***", msg)
|
|
36
|
+
|
|
30
37
|
|
|
31
38
|
def _tavily_search(query: str, max_results: int = 5) -> list[dict]:
|
|
32
39
|
api_key = os.environ.get("TAVILY_API_KEY")
|
|
33
40
|
if not api_key:
|
|
34
41
|
return []
|
|
35
|
-
|
|
36
|
-
import requests
|
|
42
|
+
import requests
|
|
37
43
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
44
|
+
resp = requests.post(
|
|
45
|
+
"https://api.tavily.com/search",
|
|
46
|
+
json={
|
|
47
|
+
"api_key": api_key,
|
|
48
|
+
"query": query,
|
|
49
|
+
"max_results": max_results,
|
|
50
|
+
"search_depth": "basic",
|
|
51
|
+
},
|
|
52
|
+
timeout=15,
|
|
53
|
+
)
|
|
54
|
+
if not resp.ok:
|
|
55
|
+
raise RuntimeError(f"tavily HTTP {resp.status_code}")
|
|
56
|
+
data = resp.json()
|
|
57
|
+
return [
|
|
58
|
+
{
|
|
59
|
+
"title": r.get("title", ""),
|
|
60
|
+
"url": r.get("url", ""),
|
|
61
|
+
"snippet": r.get("content", "")[:200],
|
|
62
|
+
"source": "tavily",
|
|
63
|
+
}
|
|
64
|
+
for r in data.get("results", [])
|
|
65
|
+
]
|
|
60
66
|
|
|
61
67
|
|
|
62
68
|
def _brave_search(query: str, count: int = 5) -> list[dict]:
|
|
63
69
|
api_key = os.environ.get("BRAVE_API_KEY")
|
|
64
70
|
if not api_key:
|
|
65
71
|
return []
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
]
|
|
85
|
-
|
|
86
|
-
return []
|
|
72
|
+
import requests
|
|
73
|
+
|
|
74
|
+
resp = requests.get(
|
|
75
|
+
"https://api.search.brave.com/res/v1/web/search",
|
|
76
|
+
headers={"X-Subscription-Token": api_key, "Accept": "application/json"},
|
|
77
|
+
params={"q": query, "count": count},
|
|
78
|
+
timeout=15,
|
|
79
|
+
)
|
|
80
|
+
if not resp.ok:
|
|
81
|
+
raise RuntimeError(f"brave HTTP {resp.status_code}")
|
|
82
|
+
data = resp.json()
|
|
83
|
+
return [
|
|
84
|
+
{
|
|
85
|
+
"title": r.get("title", ""),
|
|
86
|
+
"url": r.get("url", ""),
|
|
87
|
+
"snippet": r.get("description", "")[:200],
|
|
88
|
+
"source": "brave",
|
|
89
|
+
}
|
|
90
|
+
for r in data.get("web", {}).get("results", [])
|
|
91
|
+
]
|
|
87
92
|
|
|
88
93
|
|
|
89
94
|
def _serpapi_search(query: str, num: int = 5) -> list[dict]:
|
|
90
95
|
api_key = os.environ.get("SERPAPI_KEY")
|
|
91
96
|
if not api_key:
|
|
92
97
|
return []
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
]
|
|
116
|
-
|
|
117
|
-
return []
|
|
98
|
+
import requests
|
|
99
|
+
|
|
100
|
+
resp = requests.get(
|
|
101
|
+
"https://serpapi.com/search",
|
|
102
|
+
params={
|
|
103
|
+
"api_key": api_key,
|
|
104
|
+
"q": query,
|
|
105
|
+
"num": num,
|
|
106
|
+
"engine": "google",
|
|
107
|
+
},
|
|
108
|
+
timeout=15,
|
|
109
|
+
)
|
|
110
|
+
if not resp.ok:
|
|
111
|
+
raise RuntimeError(f"serpapi HTTP {resp.status_code}")
|
|
112
|
+
data = resp.json()
|
|
113
|
+
return [
|
|
114
|
+
{
|
|
115
|
+
"title": r.get("title", ""),
|
|
116
|
+
"url": r.get("link", ""),
|
|
117
|
+
"snippet": r.get("snippet", "")[:200],
|
|
118
|
+
"source": "serpapi",
|
|
119
|
+
}
|
|
120
|
+
for r in data.get("organic_results", [])
|
|
121
|
+
]
|
|
118
122
|
|
|
119
123
|
|
|
120
124
|
def _bocha_search(query: str, max_results: int = 5) -> list[dict]:
|
|
121
125
|
api_key = os.environ.get("BOCHA_API_KEY")
|
|
122
126
|
if not api_key:
|
|
123
127
|
return []
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
)
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
128
|
+
import requests
|
|
129
|
+
|
|
130
|
+
resp = requests.post(
|
|
131
|
+
"https://api.bochaai.com/v1/web-search",
|
|
132
|
+
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
|
133
|
+
json={"query": query, "count": max_results},
|
|
134
|
+
timeout=15,
|
|
135
|
+
)
|
|
136
|
+
if not resp.ok:
|
|
137
|
+
raise RuntimeError(f"bocha HTTP {resp.status_code}")
|
|
138
|
+
data = resp.json()
|
|
139
|
+
code = data.get("code")
|
|
140
|
+
if code is not None and str(code) != "200":
|
|
141
|
+
# Bocha can signal quota/auth failure via HTTP 200 + non-200 envelope code
|
|
142
|
+
raise RuntimeError(f"bocha code {code}: {data.get('msg') or data.get('message') or 'unknown'}")
|
|
143
|
+
# Bocha web-search returns Bing-style data.webPages.value[] (issue #13)
|
|
144
|
+
items = ((data.get("data") or {}).get("webPages") or {}).get("value") or []
|
|
145
|
+
return [
|
|
146
|
+
{
|
|
147
|
+
"title": item.get("name", ""),
|
|
148
|
+
"url": item.get("url", ""),
|
|
149
|
+
"snippet": (item.get("snippet") or "")[:200],
|
|
150
|
+
"source": "bocha",
|
|
151
|
+
}
|
|
152
|
+
for item in items
|
|
153
|
+
]
|
|
147
154
|
|
|
148
155
|
|
|
149
156
|
def _searxng_search(query: str, max_results: int = 5) -> list[dict]:
|
|
@@ -155,10 +162,10 @@ def _searxng_search(query: str, max_results: int = 5) -> list[dict]:
|
|
|
155
162
|
base_urls = [u.strip().rstrip("/") for u in os.environ.get("SEARXNG_BASE_URLS", "").split(",") if u.strip()]
|
|
156
163
|
if not base_urls:
|
|
157
164
|
return []
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
165
|
+
import requests
|
|
166
|
+
|
|
167
|
+
last_error = None
|
|
168
|
+
responded = False
|
|
162
169
|
for base in base_urls:
|
|
163
170
|
try:
|
|
164
171
|
resp = requests.get(
|
|
@@ -168,7 +175,9 @@ def _searxng_search(query: str, max_results: int = 5) -> list[dict]:
|
|
|
168
175
|
timeout=15,
|
|
169
176
|
)
|
|
170
177
|
if not resp.ok:
|
|
178
|
+
last_error = f"searxng {base} HTTP {resp.status_code}"
|
|
171
179
|
continue
|
|
180
|
+
responded = True
|
|
172
181
|
data = resp.json()
|
|
173
182
|
results = [
|
|
174
183
|
{
|
|
@@ -181,8 +190,10 @@ def _searxng_search(query: str, max_results: int = 5) -> list[dict]:
|
|
|
181
190
|
]
|
|
182
191
|
if results:
|
|
183
192
|
return results
|
|
184
|
-
except Exception:
|
|
185
|
-
|
|
193
|
+
except Exception as e:
|
|
194
|
+
last_error = f"searxng {base}: {e}"
|
|
195
|
+
if last_error and not responded:
|
|
196
|
+
raise RuntimeError(last_error)
|
|
186
197
|
return []
|
|
187
198
|
|
|
188
199
|
|
|
@@ -190,6 +201,7 @@ def search_news(query: str, max_results: int = 10) -> dict:
|
|
|
190
201
|
results = []
|
|
191
202
|
engines_tried = []
|
|
192
203
|
engines_available = []
|
|
204
|
+
engines_errors = {}
|
|
193
205
|
|
|
194
206
|
for name, fn in [
|
|
195
207
|
("tavily", _tavily_search),
|
|
@@ -199,7 +211,11 @@ def search_news(query: str, max_results: int = 10) -> dict:
|
|
|
199
211
|
("searxng", _searxng_search),
|
|
200
212
|
]:
|
|
201
213
|
engines_tried.append(name)
|
|
202
|
-
|
|
214
|
+
try:
|
|
215
|
+
items = fn(query, max_results)
|
|
216
|
+
except Exception as e:
|
|
217
|
+
engines_errors[name] = _scrub_secrets(str(e))
|
|
218
|
+
continue
|
|
203
219
|
if items:
|
|
204
220
|
engines_available.append(name)
|
|
205
221
|
results.extend(items)
|
|
@@ -207,14 +223,19 @@ def search_news(query: str, max_results: int = 10) -> dict:
|
|
|
207
223
|
break
|
|
208
224
|
|
|
209
225
|
if not results:
|
|
226
|
+
if engines_errors:
|
|
227
|
+
note = "All configured search engines failed; see engines_errors for per-engine reasons."
|
|
228
|
+
else:
|
|
229
|
+
note = (
|
|
230
|
+
"No search engines configured. Set TAVILY_API_KEY, BRAVE_API_KEY, BOCHA_API_KEY, "
|
|
231
|
+
"SERPAPI_KEY, or SEARXNG_BASE_URLS in environment."
|
|
232
|
+
)
|
|
210
233
|
return {
|
|
211
234
|
"query": query,
|
|
212
235
|
"results": [],
|
|
213
236
|
"engines_tried": engines_tried,
|
|
214
|
-
"
|
|
215
|
-
|
|
216
|
-
"SERPAPI_KEY, or SEARXNG_BASE_URLS in environment."
|
|
217
|
-
),
|
|
237
|
+
"engines_errors": engines_errors,
|
|
238
|
+
"note": note,
|
|
218
239
|
}
|
|
219
240
|
|
|
220
241
|
seen_urls = set()
|
|
@@ -228,6 +249,7 @@ def search_news(query: str, max_results: int = 10) -> dict:
|
|
|
228
249
|
"query": query,
|
|
229
250
|
"results": deduped[:max_results],
|
|
230
251
|
"engines_used": engines_available,
|
|
252
|
+
"engines_errors": engines_errors,
|
|
231
253
|
"total": len(deduped),
|
|
232
254
|
}
|
|
233
255
|
|
|
@@ -244,11 +266,13 @@ def search_comprehensive(symbol: str, name: str = None) -> dict:
|
|
|
244
266
|
}
|
|
245
267
|
|
|
246
268
|
results = {}
|
|
269
|
+
engines_errors = {}
|
|
247
270
|
for dim, query in dimensions.items():
|
|
248
271
|
data = search_news(query, max_results=3)
|
|
249
272
|
results[dim] = data.get("results", [])
|
|
273
|
+
engines_errors.update(data.get("engines_errors") or {})
|
|
250
274
|
|
|
251
|
-
return {"symbol": symbol, "name": label, "dimensions": results}
|
|
275
|
+
return {"symbol": symbol, "name": label, "dimensions": results, "engines_errors": engines_errors}
|
|
252
276
|
|
|
253
277
|
|
|
254
278
|
# --------------- A-share Sentiment Sources ---------------
|