@weaxs/openclaw-stock-analysis 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.js +322 -21
- package/index.ts +257 -16
- package/openclaw.plugin.json +11 -1
- package/package.json +1 -1
- package/schemas/market_review_schema.json +12 -5
- package/schemas/report_schema.json +68 -4
- package/scripts/cli-timeout.mjs +27 -0
- package/scripts/json-input.mjs +26 -0
- package/scripts/setup-python.mjs +6 -6
- package/skills/bull-trend/SKILL.md +34 -10
- package/skills/bull-trend/references/fat-tails.md +91 -0
- package/skills/bull-trend/scripts/tail_analysis.py +132 -0
- package/skills/event-driven/SKILL.md +6 -0
- package/skills/growth-quality/SKILL.md +6 -0
- package/skills/market-review/SKILL.md +2 -0
- package/skills/stock-analysis/SKILL.md +25 -5
- package/skills/stock-analysis/references/stateless-analysis.md +66 -0
- package/skills/stock-screener/SKILL.md +7 -1
- package/skills/strategy-backtest/SKILL.md +13 -0
- package/skills/wisburg-research/SKILL.md +2 -0
- package/templates/market_review/full.md.j2 +7 -0
- package/templates/stock_analysis/_evidence.md.j2 +11 -0
- package/templates/stock_analysis/brief.md.j2 +6 -1
- package/templates/stock_analysis/full.md.j2 +8 -3
- package/tools/_subproc.py +66 -11
- package/tools/alert_rules.py +32 -4
- package/tools/backtest.py +95 -18
- package/tools/capabilities.py +11 -0
- package/tools/cli_budgets.json +26 -0
- package/tools/gather.py +24 -16
- package/tools/lixinger.py +100 -0
- package/tools/market_regime.py +7 -3
- package/tools/market_review.py +37 -15
- package/tools/position_context.py +20 -2
- package/tools/report_renderer.py +2 -0
- package/tools/risk_screening.py +45 -17
- package/tools/screener.py +74 -24
- package/tools/sec_edgar.py +346 -0
- package/tools/sec_holders.py +263 -0
- package/tools/signal_tracker.py +218 -24
- package/tools/stateless_analysis.py +630 -0
- package/tools/stock_data.py +170 -32
- package/tools/technical.py +16 -2
- package/tools/value_data.py +159 -0
- package/tools/watchlist.py +24 -1
- package/tools/watchlist_context.py +25 -10
package/tools/screener.py
CHANGED
|
@@ -18,7 +18,7 @@ import yaml
|
|
|
18
18
|
|
|
19
19
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
20
20
|
|
|
21
|
-
from _subproc import run_tool, utf8_stdio
|
|
21
|
+
from _subproc import command_timeout, run_tool, utf8_stdio
|
|
22
22
|
from market_review import calc_temperature
|
|
23
23
|
from stock_data import compute_market_stats
|
|
24
24
|
|
|
@@ -35,6 +35,8 @@ DEFAULT_FILTERS = {
|
|
|
35
35
|
"volume_ratio_min": 0.5,
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
L1_WEIGHTS = {"value": 0.4, "momentum": 0.3, "liquidity": 0.3}
|
|
39
|
+
|
|
38
40
|
L2_WEIGHTS = {
|
|
39
41
|
"value": 0.15,
|
|
40
42
|
"quality": 0.20,
|
|
@@ -48,15 +50,17 @@ L2_WEIGHTS = {
|
|
|
48
50
|
_INDEX_REGION = {"A": "cn", "HK": "hk", "US": "us"}
|
|
49
51
|
|
|
50
52
|
|
|
51
|
-
def _fetch_json(args: list[str], timeout: int =
|
|
52
|
-
"""
|
|
53
|
-
return run_tool("stock_data.py", args, timeout=timeout)
|
|
53
|
+
def _fetch_json(args: list[str], timeout: int | None = None) -> dict | list | None:
|
|
54
|
+
"""Fetch JSON with the shared command budget and preserve structured failures."""
|
|
55
|
+
return run_tool("stock_data.py", args, timeout=timeout, preserve_errors=True)
|
|
54
56
|
|
|
55
57
|
|
|
56
58
|
def fetch_snapshot(market: str) -> list:
|
|
57
59
|
# a cold full-market snapshot can exceed 30s (CI runners) — allow longer; with
|
|
58
60
|
# all eastmoney/sina legs timing out, the tencent fallback leg needs ~2min more
|
|
59
|
-
return _fetch_json(
|
|
61
|
+
return _fetch_json(
|
|
62
|
+
["market_snapshot", "--market", market], timeout=command_timeout("stock_data.py", ["market_snapshot"])
|
|
63
|
+
)
|
|
60
64
|
|
|
61
65
|
|
|
62
66
|
def load_config(path: str) -> dict:
|
|
@@ -106,9 +110,10 @@ def apply_filters(df: pd.DataFrame, filters: dict) -> pd.DataFrame:
|
|
|
106
110
|
|
|
107
111
|
|
|
108
112
|
def _minmax(s: pd.Series) -> pd.Series:
|
|
113
|
+
s = s.where(np.isfinite(s))
|
|
109
114
|
mn, mx = s.min(), s.max()
|
|
110
115
|
if mx == mn:
|
|
111
|
-
return pd.Series(50.0, index=s.index)
|
|
116
|
+
return pd.Series(50.0, index=s.index).where(s.notna())
|
|
112
117
|
return ((s - mn) / (mx - mn) * 100).clip(0, 100)
|
|
113
118
|
|
|
114
119
|
|
|
@@ -131,7 +136,7 @@ def compute_scores(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
131
136
|
if "dividend_yield" in df.columns:
|
|
132
137
|
df["dividend_score"] = _minmax(df["dividend_yield"])
|
|
133
138
|
value_parts.append(df["dividend_score"])
|
|
134
|
-
df["value_score"] = pd.concat(value_parts, axis=1).mean(axis=1) if value_parts else
|
|
139
|
+
df["value_score"] = pd.concat(value_parts, axis=1).mean(axis=1) if value_parts else np.nan
|
|
135
140
|
|
|
136
141
|
# Momentum: higher change_pct and volume_ratio → higher score
|
|
137
142
|
mom_parts = []
|
|
@@ -139,7 +144,7 @@ def compute_scores(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
139
144
|
mom_parts.append(_minmax(df["change_pct"]))
|
|
140
145
|
if "volume_ratio" in df.columns:
|
|
141
146
|
mom_parts.append(_minmax(df["volume_ratio"]))
|
|
142
|
-
df["momentum_score"] =
|
|
147
|
+
df["momentum_score"] = pd.concat(mom_parts, axis=1).mean(axis=1) if mom_parts else np.nan
|
|
143
148
|
|
|
144
149
|
# Liquidity: higher turnover_rate and volume → higher score
|
|
145
150
|
liq_parts = []
|
|
@@ -147,9 +152,11 @@ def compute_scores(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
147
152
|
liq_parts.append(_minmax(df["turnover_rate"]))
|
|
148
153
|
if "volume" in df.columns:
|
|
149
154
|
liq_parts.append(_minmax(df["volume"].astype(float)))
|
|
150
|
-
df["liquidity_score"] =
|
|
155
|
+
df["liquidity_score"] = pd.concat(liq_parts, axis=1).mean(axis=1) if liq_parts else np.nan
|
|
151
156
|
|
|
152
|
-
|
|
157
|
+
groups = df[[f"{group}_score" for group in L1_WEIGHTS]].rename(columns=lambda name: name.removesuffix("_score"))
|
|
158
|
+
weights = pd.Series(L1_WEIGHTS)
|
|
159
|
+
df["composite_score"] = groups.mul(weights).sum(axis=1, min_count=1) / groups.notna().mul(weights).sum(axis=1)
|
|
153
160
|
return df
|
|
154
161
|
|
|
155
162
|
|
|
@@ -298,8 +305,9 @@ def _apply_l2(candidates: list, market: str, multiplier: float) -> None:
|
|
|
298
305
|
scores = c["scores"]
|
|
299
306
|
# scores are sanitized to None when non-finite; L2 math needs numbers
|
|
300
307
|
components = {
|
|
301
|
-
|
|
302
|
-
"
|
|
308
|
+
group: scores[group]
|
|
309
|
+
for group in ("value", "liquidity")
|
|
310
|
+
if group in c["factor_coverage"]["available"] and scores[group] is not None
|
|
303
311
|
}
|
|
304
312
|
# momentum is L2-only: candidates without kline factors drop the group and renormalize weights.
|
|
305
313
|
components.update(
|
|
@@ -309,25 +317,35 @@ def _apply_l2(candidates: list, market: str, multiplier: float) -> None:
|
|
|
309
317
|
{g: round(groups[g], 1) for g in ("quality", "growth", "moneyflow", "low_volatility") if g in groups}
|
|
310
318
|
)
|
|
311
319
|
total_weight = sum(L2_WEIGHTS[g] for g in components)
|
|
312
|
-
composite_l2 = sum(L2_WEIGHTS[g] * components[g] for g in components) / total_weight
|
|
313
|
-
scores["final"] = round(composite_l2 * multiplier, 1)
|
|
320
|
+
composite_l2 = sum(L2_WEIGHTS[g] * components[g] for g in components) / total_weight if total_weight else None
|
|
321
|
+
scores["final"] = round(composite_l2 * multiplier, 1) if composite_l2 is not None else None
|
|
322
|
+
c["factor_coverage"] = {
|
|
323
|
+
"available": list(components),
|
|
324
|
+
"missing": [g for g in L2_WEIGHTS if g not in components],
|
|
325
|
+
"ratio": round(total_weight, 4),
|
|
326
|
+
"effective_weights": {g: L2_WEIGHTS[g] / total_weight for g in components},
|
|
327
|
+
"effective_scores": components,
|
|
328
|
+
}
|
|
314
329
|
|
|
315
330
|
|
|
316
331
|
def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict:
|
|
332
|
+
if isinstance(top, bool) or not isinstance(top, int) or top <= 0:
|
|
333
|
+
return {"error": "top must be a positive integer", "market": market}
|
|
334
|
+
if config is not None and (not isinstance(config, dict) or not isinstance(config.get("filters", {}), dict)):
|
|
335
|
+
return {"error": "config and filters must be objects", "market": market}
|
|
317
336
|
filters = merge_filters(config)
|
|
337
|
+
supported = set(DEFAULT_FILTERS) | {"dividend_yield_min", "dividend_yield_max"}
|
|
338
|
+
for key, value in filters.items():
|
|
339
|
+
if key not in supported:
|
|
340
|
+
return {"error": f"Unsupported filter: {key}", "market": market}
|
|
341
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not np.isfinite(value):
|
|
342
|
+
return {"error": f"{key} must be a finite number", "market": market}
|
|
318
343
|
dividend_requested = any(k.startswith("dividend_yield_") for k in filters) or (config or {}).get("sort_by") in (
|
|
319
344
|
"dividend_yield",
|
|
320
345
|
"dividend_score",
|
|
321
346
|
)
|
|
322
347
|
if dividend_requested and market not in ("A", "HK"):
|
|
323
348
|
return {"error": "dividend screening currently covers A/HK only", "market": market}
|
|
324
|
-
for key in ("dividend_yield_min", "dividend_yield_max"):
|
|
325
|
-
if key in filters and (
|
|
326
|
-
isinstance(filters[key], bool)
|
|
327
|
-
or not isinstance(filters[key], (int, float))
|
|
328
|
-
or not np.isfinite(filters[key])
|
|
329
|
-
):
|
|
330
|
-
return {"error": f"{key} must be a finite number in percent units", "market": market}
|
|
331
349
|
raw = fetch_snapshot(market)
|
|
332
350
|
if isinstance(raw, dict) and "error" in raw:
|
|
333
351
|
return raw
|
|
@@ -378,6 +396,11 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
378
396
|
"error": "dividend data unavailable; requested dividend criterion was not applied",
|
|
379
397
|
"market": market,
|
|
380
398
|
}
|
|
399
|
+
explicit = (config or {}).get("filters") or {}
|
|
400
|
+
missing = [key for key in explicit if key.rsplit("_", 1)[0] not in df]
|
|
401
|
+
if missing:
|
|
402
|
+
return {"error": f"Requested filter data unavailable: {', '.join(missing)}", "market": market}
|
|
403
|
+
applied_filters = {key: value for key, value in filters.items() if key.rsplit("_", 1)[0] in df}
|
|
381
404
|
df = apply_filters(df, filters)
|
|
382
405
|
filtered_count = len(df)
|
|
383
406
|
|
|
@@ -387,7 +410,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
387
410
|
"total_stocks": total,
|
|
388
411
|
"filtered_count": 0,
|
|
389
412
|
"returned_count": 0,
|
|
390
|
-
"filters_applied":
|
|
413
|
+
"filters_applied": applied_filters,
|
|
391
414
|
"candidates": [],
|
|
392
415
|
"market_sentiment": sentiment,
|
|
393
416
|
"l2_enabled": l2_enabled,
|
|
@@ -437,11 +460,36 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
437
460
|
}
|
|
438
461
|
if "dividend_score" in row:
|
|
439
462
|
entry["scores"]["dividend"] = _finite(round(float(row["dividend_score"]), 1))
|
|
463
|
+
factor_inputs = {
|
|
464
|
+
"value": ("pe", "pb", "dividend_yield"),
|
|
465
|
+
"momentum": ("change_pct", "volume_ratio"),
|
|
466
|
+
"liquidity": ("turnover_rate", "volume"),
|
|
467
|
+
}
|
|
468
|
+
weights = L1_WEIGHTS
|
|
469
|
+
available = [
|
|
470
|
+
g
|
|
471
|
+
for g, fields in factor_inputs.items()
|
|
472
|
+
if entry["scores"][g] is not None and any(field in entry for field in fields)
|
|
473
|
+
]
|
|
474
|
+
for group in weights:
|
|
475
|
+
if group not in available:
|
|
476
|
+
entry["scores"][group] = None
|
|
477
|
+
if not available:
|
|
478
|
+
entry["scores"]["composite"] = entry["scores"]["final"] = None
|
|
479
|
+
total_weight = sum(weights[g] for g in available)
|
|
480
|
+
entry["factor_coverage"] = {
|
|
481
|
+
"available": available,
|
|
482
|
+
"missing": [g for g in weights if g not in available],
|
|
483
|
+
"ratio": round(total_weight, 4),
|
|
484
|
+
"effective_weights": {g: weights[g] / total_weight for g in available},
|
|
485
|
+
}
|
|
440
486
|
candidates.append(entry)
|
|
441
487
|
|
|
442
488
|
if l2_enabled:
|
|
443
489
|
_apply_l2(candidates, market, multiplier)
|
|
444
|
-
candidates.sort(
|
|
490
|
+
candidates.sort(
|
|
491
|
+
key=lambda c: c["scores"]["final"] if c["scores"]["final"] is not None else -float("inf"), reverse=True
|
|
492
|
+
)
|
|
445
493
|
for rank, entry in enumerate(candidates, 1):
|
|
446
494
|
entry["rank"] = rank
|
|
447
495
|
|
|
@@ -450,7 +498,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
450
498
|
"total_stocks": total,
|
|
451
499
|
"filtered_count": filtered_count,
|
|
452
500
|
"returned_count": len(candidates),
|
|
453
|
-
"filters_applied":
|
|
501
|
+
"filters_applied": applied_filters,
|
|
454
502
|
"candidates": candidates,
|
|
455
503
|
"market_sentiment": sentiment,
|
|
456
504
|
"l2_enabled": l2_enabled,
|
|
@@ -478,6 +526,8 @@ def main():
|
|
|
478
526
|
config = load_config(args.config) if args.config else None
|
|
479
527
|
result = screen(args.market, args.top, config, l2=args.l2)
|
|
480
528
|
print(json.dumps(result, ensure_ascii=False, default=str))
|
|
529
|
+
if "error" in result:
|
|
530
|
+
sys.exit(1)
|
|
481
531
|
|
|
482
532
|
|
|
483
533
|
if __name__ == "__main__":
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""SEC institutional 13F and issuer Form 4 disclosures, never realtime holdings."""
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
import time
|
|
11
|
+
from datetime import date, timedelta
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from xml.etree import ElementTree as ET
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
18
|
+
from _cache import DAY, WEEK, cached
|
|
19
|
+
from _subproc import utf8_stdio
|
|
20
|
+
|
|
21
|
+
_last_request = 0.0
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def identity():
|
|
25
|
+
agent = os.environ.get("SEC_USER_AGENT", "").strip()
|
|
26
|
+
if "\n" in agent or "\r" in agent or not re.search(r"[^\s@]+@[^\s@]+\.[^\s@]+", agent):
|
|
27
|
+
raise ValueError("Set SEC_USER_AGENT to your application name and real contact email")
|
|
28
|
+
return agent
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def sec_get(url, as_json=True, stream=False):
|
|
32
|
+
global _last_request
|
|
33
|
+
agent = identity()
|
|
34
|
+
time.sleep(max(0, 0.5 - (time.monotonic() - _last_request)))
|
|
35
|
+
_last_request = time.monotonic()
|
|
36
|
+
try:
|
|
37
|
+
response = requests.get(
|
|
38
|
+
url,
|
|
39
|
+
headers={"User-Agent": agent, "Accept-Encoding": "gzip, deflate"},
|
|
40
|
+
timeout=(10, 30) if stream else 20,
|
|
41
|
+
stream=stream,
|
|
42
|
+
)
|
|
43
|
+
response.raise_for_status()
|
|
44
|
+
if stream:
|
|
45
|
+
return response
|
|
46
|
+
return response.json() if as_json else response.text
|
|
47
|
+
except Exception as exc:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f"SEC data unavailable ({type(exc).__name__}); check identity/access and retry later"
|
|
50
|
+
) from None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def resolve_cik(symbol):
|
|
54
|
+
if re.fullmatch(r"\d{1,10}", symbol) and int(symbol) > 0:
|
|
55
|
+
return symbol.zfill(10)
|
|
56
|
+
if not re.fullmatch(r"[A-Za-z][A-Za-z0-9.-]{0,14}", symbol):
|
|
57
|
+
raise ValueError("A US ticker or numeric CIK is required")
|
|
58
|
+
mapping = cached("sec-tickers:US", WEEK, lambda: sec_get("https://www.sec.gov/files/company_tickers.json"))
|
|
59
|
+
for row in mapping.values():
|
|
60
|
+
if row["ticker"].upper().replace(".", "-") == symbol.upper().replace(".", "-"):
|
|
61
|
+
return str(row["cik_str"]).zfill(10)
|
|
62
|
+
raise ValueError("Ticker not found in SEC directory; supply issuer CIK")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def filing_rows(data):
|
|
66
|
+
return [
|
|
67
|
+
{key: values[i] for key, values in data.items() if isinstance(values, list) and i < len(values)}
|
|
68
|
+
for i in range(len(data.get("accessionNumber", [])))
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def submissions(cik, forms, cutoff=None):
|
|
73
|
+
data = cached(f"sec-submissions:{cik}", DAY, lambda: sec_get(f"https://data.sec.gov/submissions/CIK{cik}.json"))
|
|
74
|
+
if not isinstance(data.get("filings", {}).get("recent"), dict):
|
|
75
|
+
raise ValueError("SEC submissions metadata unavailable")
|
|
76
|
+
rows = filing_rows(data.get("filings", {}).get("recent", {}))
|
|
77
|
+
files = data.get("filings", {}).get("files", [])
|
|
78
|
+
relevant = [f for f in files if not cutoff or f.get("filingTo", "") >= cutoff]
|
|
79
|
+
relevant.sort(key=lambda f: f.get("filingTo", ""), reverse=True)
|
|
80
|
+
loaded = 0
|
|
81
|
+
# shortcut: at most three older submission shards; expose incomplete coverage when more are needed.
|
|
82
|
+
for file in relevant[:3]:
|
|
83
|
+
latest = max((r.get("reportDate") or "" for r in rows if r.get("form") in forms), default="")
|
|
84
|
+
if (
|
|
85
|
+
not cutoff
|
|
86
|
+
and file.get("filingTo", "") <= latest
|
|
87
|
+
and any(r.get("form") == "13F-HR" and (r.get("reportDate") or "") == latest for r in rows)
|
|
88
|
+
):
|
|
89
|
+
break
|
|
90
|
+
name = file.get("name", "")
|
|
91
|
+
if not re.fullmatch(r"CIK\d{10}-submissions-\d+\.json", name):
|
|
92
|
+
raise ValueError("Unexpected SEC submissions filename")
|
|
93
|
+
extra = cached(
|
|
94
|
+
f"sec-submission-file:{name}", DAY, lambda name=name: sec_get(f"https://data.sec.gov/submissions/{name}")
|
|
95
|
+
)
|
|
96
|
+
rows.extend(filing_rows(extra))
|
|
97
|
+
loaded += 1
|
|
98
|
+
result = [r for r in rows if r.get("form") in forms and (not cutoff or r.get("filingDate", "") >= cutoff)]
|
|
99
|
+
result.sort(key=lambda r: (r.get("filingDate", ""), r.get("accessionNumber", "")), reverse=True)
|
|
100
|
+
earliest_unsearched = cutoff or max((r.get("reportDate") or "" for r in result), default="")
|
|
101
|
+
incomplete = any(f.get("filingTo", "") >= earliest_unsearched for f in relevant[loaded:])
|
|
102
|
+
return data.get("name"), result, incomplete
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def archive_base(cik, filing):
|
|
106
|
+
accession = filing.get("accessionNumber", "")
|
|
107
|
+
if not re.fullmatch(r"\d{10}-\d{2}-\d{6}", accession):
|
|
108
|
+
raise ValueError("Invalid SEC accession number")
|
|
109
|
+
return f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/{accession.replace('-', '')}"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def xml_root(text):
|
|
113
|
+
if "<!DOCTYPE" in text.upper() or "<!ENTITY" in text.upper():
|
|
114
|
+
raise ValueError("Unexpected XML entity declaration")
|
|
115
|
+
root = ET.fromstring(text)
|
|
116
|
+
for element in root.iter():
|
|
117
|
+
element.tag = element.tag.rsplit("}", 1)[-1]
|
|
118
|
+
return root
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def footnote_refs(node, path=""):
|
|
122
|
+
refs = {}
|
|
123
|
+
for child in node:
|
|
124
|
+
child_path = f"{path}/{child.tag}" if path else child.tag
|
|
125
|
+
if child.tag == "footnoteId":
|
|
126
|
+
refs.setdefault(path, []).append(child.get("id"))
|
|
127
|
+
else:
|
|
128
|
+
refs.update(footnote_refs(child, child_path))
|
|
129
|
+
return refs
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def numeric(node, path):
|
|
133
|
+
value = node.findtext(path)
|
|
134
|
+
try:
|
|
135
|
+
result = float(value)
|
|
136
|
+
return result if math.isfinite(result) else None
|
|
137
|
+
except (ValueError, TypeError):
|
|
138
|
+
return None
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def holdings(args):
|
|
142
|
+
identity()
|
|
143
|
+
if not re.fullmatch(r"\d{1,10}", args.cik) or int(args.cik) == 0:
|
|
144
|
+
raise ValueError("13F requires a manager or issuer CIK (1..10 digits)")
|
|
145
|
+
cik = args.cik.zfill(10)
|
|
146
|
+
mode = getattr(args, "mode", "auto")
|
|
147
|
+
if mode not in ("auto", "manager", "issuer"):
|
|
148
|
+
raise ValueError("mode must be auto, manager or issuer")
|
|
149
|
+
if mode != "manager":
|
|
150
|
+
data = cached(f"sec-submissions:{cik}", DAY, lambda: sec_get(f"https://data.sec.gov/submissions/CIK{cik}.json"))
|
|
151
|
+
if not isinstance(data.get("filings", {}).get("recent"), dict):
|
|
152
|
+
raise ValueError("SEC submissions metadata unavailable")
|
|
153
|
+
forms = data["filings"]["recent"].get("form", [])
|
|
154
|
+
is_manager = any(form in ("13F-HR", "13F-HR/A") for form in forms)
|
|
155
|
+
if mode == "auto" and not is_manager:
|
|
156
|
+
_, archived, _ = submissions(cik, {"13F-HR", "13F-HR/A"})
|
|
157
|
+
is_manager = any(row["form"] == "13F-HR" for row in archived)
|
|
158
|
+
if mode == "issuer" or not is_manager:
|
|
159
|
+
from sec_holders import issuer_holders
|
|
160
|
+
|
|
161
|
+
return issuer_holders(cik, data)
|
|
162
|
+
name, filings, incomplete = submissions(cik, {"13F-HR", "13F-HR/A"})
|
|
163
|
+
originals = [r for r in filings if r["form"] == "13F-HR"]
|
|
164
|
+
if not originals:
|
|
165
|
+
raise ValueError("No original 13F-HR in available submission history; issuer CIK is not a holder index")
|
|
166
|
+
filing = max(originals, key=lambda r: (r.get("reportDate", ""), r.get("filingDate", "")))
|
|
167
|
+
if max((r.get("reportDate") or "" for r in filings), default="") > (filing.get("reportDate") or ""):
|
|
168
|
+
raise ValueError("Latest 13F report period has amendments but its original is unavailable in searched history")
|
|
169
|
+
amendments = [r for r in filings if r["form"] == "13F-HR/A" and r.get("reportDate") == filing.get("reportDate")]
|
|
170
|
+
base = archive_base(cik, filing)
|
|
171
|
+
|
|
172
|
+
def fetch_table():
|
|
173
|
+
listing = sec_get(f"{base}/index.json")["directory"]["item"]
|
|
174
|
+
candidates = [r["name"] for r in listing if re.fullmatch(r"[\w.-]+\.xml", r.get("name", ""), re.I)]
|
|
175
|
+
for filename in candidates[:10]:
|
|
176
|
+
try:
|
|
177
|
+
root = xml_root(sec_get(f"{base}/{filename}", as_json=False))
|
|
178
|
+
except ET.ParseError:
|
|
179
|
+
continue
|
|
180
|
+
if root.tag == "informationTable":
|
|
181
|
+
return [
|
|
182
|
+
{
|
|
183
|
+
"issuer": row.findtext("nameOfIssuer"),
|
|
184
|
+
"class": row.findtext("titleOfClass"),
|
|
185
|
+
"cusip": row.findtext("cusip"),
|
|
186
|
+
"value": numeric(row, "value"),
|
|
187
|
+
"shares": numeric(row, "shrsOrPrnAmt/sshPrnamt"),
|
|
188
|
+
"share_type": row.findtext("shrsOrPrnAmt/sshPrnamtType"),
|
|
189
|
+
"put_call": row.findtext("putCall"),
|
|
190
|
+
"investment_discretion": row.findtext("investmentDiscretion"),
|
|
191
|
+
}
|
|
192
|
+
for row in root.findall("infoTable")
|
|
193
|
+
]
|
|
194
|
+
raise ValueError("13F information table XML unavailable")
|
|
195
|
+
|
|
196
|
+
rows = cached(f"sec-13f:{cik}:{filing['accessionNumber']}", WEEK, fetch_table)
|
|
197
|
+
result = {
|
|
198
|
+
"cik": cik,
|
|
199
|
+
"manager": name,
|
|
200
|
+
"query_type": "manager",
|
|
201
|
+
"source": "sec_edgar",
|
|
202
|
+
"report_date": filing.get("reportDate"),
|
|
203
|
+
"filing_date": filing["filingDate"],
|
|
204
|
+
"accession": filing["accessionNumber"],
|
|
205
|
+
"filing_url": f"{base}/{filing['accessionNumber']}-index.html",
|
|
206
|
+
"holdings": rows,
|
|
207
|
+
"value_unit": "USD" if filing["filingDate"] >= "2023-01-03" else "USD thousands",
|
|
208
|
+
"disclosure_lag_days": (
|
|
209
|
+
date.fromisoformat(filing["filingDate"]) - date.fromisoformat(filing["reportDate"])
|
|
210
|
+
).days,
|
|
211
|
+
"filing_deadline_days": 45,
|
|
212
|
+
"data_age_days": (date.today() - date.fromisoformat(filing["reportDate"])).days,
|
|
213
|
+
"realtime": False,
|
|
214
|
+
"history_truncated": incomplete,
|
|
215
|
+
"coverage": "13F reportable securities only; not a complete portfolio or issuer-to-managers index",
|
|
216
|
+
"amendments": [
|
|
217
|
+
{
|
|
218
|
+
"accession": r["accessionNumber"],
|
|
219
|
+
"filing_date": r["filingDate"],
|
|
220
|
+
"filing_url": f"{archive_base(cik, r)}/{r['accessionNumber']}-index.html",
|
|
221
|
+
}
|
|
222
|
+
for r in amendments
|
|
223
|
+
],
|
|
224
|
+
}
|
|
225
|
+
if incomplete:
|
|
226
|
+
result["warnings"] = ["Older submission history not fully searched; latest report period may be incomplete"]
|
|
227
|
+
if amendments:
|
|
228
|
+
result.setdefault("warnings", []).append(
|
|
229
|
+
"Amendments exist; original holdings returned, review amendment links before comparison"
|
|
230
|
+
)
|
|
231
|
+
return result
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def form4(cik, filing):
|
|
235
|
+
base = archive_base(cik, filing)
|
|
236
|
+
document = filing.get("primaryDocument", "").rsplit("/", 1)[-1]
|
|
237
|
+
if not re.fullmatch(r"[\w.-]+\.xml", document, re.I):
|
|
238
|
+
raise ValueError("Form 4 primary XML unavailable")
|
|
239
|
+
root = xml_root(sec_get(f"{base}/{document}", as_json=False))
|
|
240
|
+
if root.tag != "ownershipDocument" or int(root.findtext("issuer/issuerCik") or 0) != int(cik):
|
|
241
|
+
raise ValueError("Form 4 issuer mismatch")
|
|
242
|
+
owners = [
|
|
243
|
+
{
|
|
244
|
+
"name": r.findtext("reportingOwnerId/rptOwnerName"),
|
|
245
|
+
"cik": r.findtext("reportingOwnerId/rptOwnerCik"),
|
|
246
|
+
"officer_title": r.findtext("reportingOwnerRelationship/officerTitle"),
|
|
247
|
+
"is_director": r.findtext("reportingOwnerRelationship/isDirector") in ("1", "true"),
|
|
248
|
+
"is_officer": r.findtext("reportingOwnerRelationship/isOfficer") in ("1", "true"),
|
|
249
|
+
"is_ten_percent_owner": r.findtext("reportingOwnerRelationship/isTenPercentOwner") in ("1", "true"),
|
|
250
|
+
}
|
|
251
|
+
for r in root.findall("reportingOwner")
|
|
252
|
+
]
|
|
253
|
+
footnotes = {r.get("id"): "".join(r.itertext()) for r in root.findall("footnotes/footnote")}
|
|
254
|
+
rows = []
|
|
255
|
+
for table, tag, derivative in [
|
|
256
|
+
("nonDerivativeTable", "nonDerivativeTransaction", False),
|
|
257
|
+
("derivativeTable", "derivativeTransaction", True),
|
|
258
|
+
]:
|
|
259
|
+
for row in root.findall(f"{table}/{tag}"):
|
|
260
|
+
code = row.findtext("transactionCoding/transactionCode")
|
|
261
|
+
rows.append(
|
|
262
|
+
{
|
|
263
|
+
"transaction_date": row.findtext("transactionDate/value"),
|
|
264
|
+
"owners": owners,
|
|
265
|
+
"security": row.findtext("securityTitle/value"),
|
|
266
|
+
"derivative": derivative,
|
|
267
|
+
"transaction_code": code,
|
|
268
|
+
"direction": {"P": "buy", "S": "sell"}.get(code, "other"),
|
|
269
|
+
"acquired_disposed": row.findtext("transactionAmounts/transactionAcquiredDisposedCode/value"),
|
|
270
|
+
"shares": numeric(row, "transactionAmounts/transactionShares/value"),
|
|
271
|
+
"price": numeric(row, "transactionAmounts/transactionPricePerShare/value"),
|
|
272
|
+
"shares_after": numeric(row, "postTransactionAmounts/sharesOwnedFollowingTransaction/value"),
|
|
273
|
+
"ownership": row.findtext("ownershipNature/directOrIndirectOwnership/value"),
|
|
274
|
+
"footnotes": footnotes,
|
|
275
|
+
"footnote_refs": footnote_refs(row),
|
|
276
|
+
"filing_date": filing["filingDate"],
|
|
277
|
+
"form": filing["form"],
|
|
278
|
+
"accession": filing["accessionNumber"],
|
|
279
|
+
"filing_url": f"{base}/{document}",
|
|
280
|
+
}
|
|
281
|
+
)
|
|
282
|
+
return rows
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def insider_trades(args):
|
|
286
|
+
identity()
|
|
287
|
+
if not 1 <= args.days <= 365 or not 1 <= args.top <= 20:
|
|
288
|
+
raise ValueError("days must be in 1..365 and top (filings) in 1..20")
|
|
289
|
+
cik = resolve_cik(args.symbol)
|
|
290
|
+
cutoff = (date.today() - timedelta(days=args.days)).isoformat()
|
|
291
|
+
_, filings, incomplete = submissions(cik, {"4", "4/A"}, cutoff)
|
|
292
|
+
result = {
|
|
293
|
+
"symbol": args.symbol,
|
|
294
|
+
"cik": cik,
|
|
295
|
+
"source": "sec_edgar",
|
|
296
|
+
"since_filing_date": cutoff,
|
|
297
|
+
"transactions": [],
|
|
298
|
+
"filings_scanned": 0,
|
|
299
|
+
"truncated": incomplete or len(filings) > args.top,
|
|
300
|
+
"coverage": "issuer Form 4 filings in filing-date window; top limits filings, not transactions",
|
|
301
|
+
}
|
|
302
|
+
for filing in filings[: args.top]:
|
|
303
|
+
try:
|
|
304
|
+
rows = cached(
|
|
305
|
+
f"sec-form4:{cik}:{filing['accessionNumber']}", WEEK, lambda filing=filing: form4(cik, filing)
|
|
306
|
+
)
|
|
307
|
+
result["transactions"].extend(rows)
|
|
308
|
+
result["filings_scanned"] += 1
|
|
309
|
+
except Exception as exc:
|
|
310
|
+
result.setdefault("warnings", []).append(f"{filing['accessionNumber']}: unavailable ({type(exc).__name__})")
|
|
311
|
+
if filings and not result["filings_scanned"]:
|
|
312
|
+
raise ValueError("All selected Form 4 XML documents unavailable")
|
|
313
|
+
if any(r["form"] == "4/A" for r in result["transactions"]):
|
|
314
|
+
result.setdefault("warnings", []).append(
|
|
315
|
+
"Includes Form 4 amendments; do not double-count original and amended transactions"
|
|
316
|
+
)
|
|
317
|
+
return result
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def main():
|
|
321
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
322
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
323
|
+
p = sub.add_parser("holdings")
|
|
324
|
+
p.add_argument("cik")
|
|
325
|
+
p.add_argument("--mode", choices=["auto", "manager", "issuer"], default="auto")
|
|
326
|
+
p = sub.add_parser("insider_trades")
|
|
327
|
+
p.add_argument("symbol")
|
|
328
|
+
p.add_argument("--days", type=int, default=90)
|
|
329
|
+
p.add_argument("--top", type=int, default=10)
|
|
330
|
+
try:
|
|
331
|
+
args = parser.parse_args()
|
|
332
|
+
result = holdings(args) if args.command == "holdings" else insider_trades(args)
|
|
333
|
+
print(json.dumps(result, ensure_ascii=False, allow_nan=False))
|
|
334
|
+
except SystemExit as exc:
|
|
335
|
+
if exc.code:
|
|
336
|
+
print(json.dumps({"error": "invalid CLI arguments"}))
|
|
337
|
+
raise
|
|
338
|
+
except Exception as exc:
|
|
339
|
+
message = str(exc) if isinstance(exc, ValueError) else f"SEC data unavailable ({type(exc).__name__})"
|
|
340
|
+
print(json.dumps({"error": message}, ensure_ascii=False))
|
|
341
|
+
sys.exit(1)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
if __name__ == "__main__":
|
|
345
|
+
utf8_stdio()
|
|
346
|
+
main()
|