dsh-stock-analysis 0.2.11 → 0.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -79
- package/dist/index.js +3782 -2693
- package/dist/web/app.js +38 -6
- package/dist/web/style.css +1 -0
- package/package.json +1 -1
- package/scripts/cli-json.mjs +23 -0
- package/scripts/exec-process.mjs +89 -0
- package/scripts/setup-python.mjs +37 -4
- package/scripts/tool-calls.mjs +489 -0
- package/skills/event-driven/SKILL.md +2 -2
- package/skills/growth-quality/SKILL.md +1 -1
- package/skills/stock-analysis/SKILL.md +8 -6
- package/skills/stock-analysis/references/stateless-analysis.md +9 -1
- package/tools/_cache.py +13 -0
- package/tools/_subproc.py +46 -0
- package/tools/alert_rules.py +2 -2
- package/tools/backtest.py +3 -3
- package/tools/capabilities.py +32 -9
- package/tools/cli_budgets.json +2 -0
- package/tools/diagnostics.py +3 -4
- package/tools/gather.py +2 -6
- package/tools/import_parser.py +3 -4
- package/tools/lixinger.py +119 -27
- package/tools/market_regime.py +5 -2
- package/tools/market_review.py +5 -6
- package/tools/name_resolver.py +66 -7
- package/tools/report_renderer.py +3 -3
- package/tools/requirements.txt +5 -4
- package/tools/risk_screening.py +3 -2
- package/tools/screener.py +6 -4
- package/tools/search_intel.py +79 -10
- package/tools/sec_edgar.py +266 -0
- package/tools/sec_holders.py +6 -0
- package/tools/signal_tracker.py +5 -5
- package/tools/stateless_analysis.py +115 -0
- package/tools/stock_data.py +455 -201
- package/tools/tiaa_disclosures.py +4 -0
- package/tools/tool_schemas.json +2704 -0
- package/tools/trading_calendar.py +65 -30
- package/tools/value_data.py +366 -134
- package/tools/watchlist.py +43 -5
package/tools/name_resolver.py
CHANGED
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
"""Resolve Chinese stock names / pinyin / fuzzy input to stock codes."""
|
|
3
3
|
|
|
4
4
|
import argparse
|
|
5
|
+
import functools
|
|
5
6
|
import json
|
|
7
|
+
import re
|
|
6
8
|
import sys
|
|
7
9
|
from difflib import SequenceMatcher
|
|
8
10
|
from pathlib import Path
|
|
@@ -10,8 +12,8 @@ from pathlib import Path
|
|
|
10
12
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
11
13
|
|
|
12
14
|
from _cache import DAY, cached # noqa: E402
|
|
13
|
-
from _subproc import utf8_stdio # noqa: E402
|
|
14
|
-
from stock_data import _akshare_retry # noqa: E402
|
|
15
|
+
from _subproc import error_message, utf8_stdio # noqa: E402
|
|
16
|
+
from stock_data import _akshare_retry, normalize_us_symbol # noqa: E402
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
def _load_stock_map() -> list[dict]:
|
|
@@ -39,6 +41,39 @@ def _load_stock_map() -> list[dict]:
|
|
|
39
41
|
raise RuntimeError(f"Failed to load stock list: {e}") from e
|
|
40
42
|
|
|
41
43
|
|
|
44
|
+
@functools.cache # one attempt per process: import_parser resolves many names and failures are not disk-cached
|
|
45
|
+
def _load_international_stock_map() -> list[dict]:
|
|
46
|
+
# Odd provider rows are skipped: one unexpected code must not drop a whole market's names.
|
|
47
|
+
def fetch_hk():
|
|
48
|
+
import akshare as ak
|
|
49
|
+
|
|
50
|
+
records = []
|
|
51
|
+
for row in _akshare_retry(ak.stock_hk_spot_em).to_dict("records"):
|
|
52
|
+
code, name = str(row["代码"]).split(".", 1)[0].zfill(5), row["名称"]
|
|
53
|
+
if code.isdigit() and len(code) == 5 and isinstance(name, str) and name.strip():
|
|
54
|
+
records.append({"code": f"{code}.HK", "name": name})
|
|
55
|
+
return records
|
|
56
|
+
|
|
57
|
+
def fetch_us():
|
|
58
|
+
import akshare as ak
|
|
59
|
+
|
|
60
|
+
records = []
|
|
61
|
+
for row in _akshare_retry(ak.stock_us_spot_em).to_dict("records"):
|
|
62
|
+
code, name = normalize_us_symbol(row["代码"]), row["名称"]
|
|
63
|
+
if re.fullmatch(r"[A-Z0-9-]{1,10}", code) and isinstance(name, str) and name.strip():
|
|
64
|
+
records.append({"code": code, "name": name})
|
|
65
|
+
return records
|
|
66
|
+
|
|
67
|
+
records = []
|
|
68
|
+
for key, fetch in (("stock-name-map:hk:v1", fetch_hk), ("stock-name-map:us:v1", fetch_us)):
|
|
69
|
+
try:
|
|
70
|
+
records.extend(cached(key, DAY, fetch))
|
|
71
|
+
except Exception:
|
|
72
|
+
# One unavailable market must not hide the other markets' names.
|
|
73
|
+
continue
|
|
74
|
+
return records
|
|
75
|
+
|
|
76
|
+
|
|
42
77
|
def _to_pinyin(text: str) -> str:
|
|
43
78
|
try:
|
|
44
79
|
from pypinyin import lazy_pinyin
|
|
@@ -57,16 +92,16 @@ def _to_pinyin_initials(text: str) -> str:
|
|
|
57
92
|
return ""
|
|
58
93
|
|
|
59
94
|
|
|
60
|
-
def
|
|
61
|
-
stocks = _load_stock_map()
|
|
95
|
+
def _resolve_stocks(query: str, top: int, stocks: list[dict]) -> list[dict]:
|
|
62
96
|
query = query.strip()
|
|
63
97
|
q_lower = query.lower()
|
|
98
|
+
q_code = query.upper()
|
|
64
99
|
|
|
65
|
-
exact = [s for s in stocks if s["code"] ==
|
|
100
|
+
exact = [s for s in stocks if s["code"].upper() == q_code or s["name"] == query]
|
|
66
101
|
if exact:
|
|
67
102
|
return exact[:top]
|
|
68
103
|
|
|
69
|
-
prefix_code = [s for s in stocks if s["code"].startswith(
|
|
104
|
+
prefix_code = [s for s in stocks if s["code"].upper().startswith(q_code)]
|
|
70
105
|
if prefix_code:
|
|
71
106
|
return prefix_code[:top]
|
|
72
107
|
|
|
@@ -104,6 +139,30 @@ def resolve(query: str, top: int = 5) -> list[dict]:
|
|
|
104
139
|
return [{"code": s["code"], "name": s["name"], "score": round(sc, 2)} for sc, s in scored[:top]]
|
|
105
140
|
|
|
106
141
|
|
|
142
|
+
def resolve(query: str, top: int = 5) -> list[dict]:
|
|
143
|
+
load_error = None
|
|
144
|
+
try:
|
|
145
|
+
stocks = _load_stock_map()
|
|
146
|
+
local = _resolve_stocks(query, top, stocks)
|
|
147
|
+
except Exception as exc:
|
|
148
|
+
stocks, local, load_error = [], [], exc
|
|
149
|
+
query = query.strip()
|
|
150
|
+
code_like = re.fullmatch(r"[A-Za-z][A-Za-z0-9.-]{0,14}", query)
|
|
151
|
+
# A substring hit (京东 → 京东方A) is not proof the user meant an A share (京东集团-SW is HK).
|
|
152
|
+
partial_name = bool(local) and "score" not in local[0] and query != local[0]["name"] and query in local[0]["name"]
|
|
153
|
+
# Pinyin/initials scores are not proof either (特斯拉 → 天士力 scores 0.9), so only literal hits short-circuit.
|
|
154
|
+
if local and "score" not in local[0] and not code_like and not partial_name:
|
|
155
|
+
return local
|
|
156
|
+
international = _load_international_stock_map()
|
|
157
|
+
if international:
|
|
158
|
+
hits = _resolve_stocks(query, top, stocks + international)
|
|
159
|
+
if load_error is None or (hits and "score" not in hits[0]): # fuzzy-only hits are a miss
|
|
160
|
+
return hits
|
|
161
|
+
if load_error is not None:
|
|
162
|
+
raise load_error
|
|
163
|
+
return local
|
|
164
|
+
|
|
165
|
+
|
|
107
166
|
def main():
|
|
108
167
|
parser = argparse.ArgumentParser(description="Stock name resolver")
|
|
109
168
|
sub = parser.add_subparsers(dest="command")
|
|
@@ -120,7 +179,7 @@ def main():
|
|
|
120
179
|
try:
|
|
121
180
|
result = resolve(args.query, args.top)
|
|
122
181
|
except RuntimeError as e:
|
|
123
|
-
result = {"error":
|
|
182
|
+
result = {"error": error_message(e)}
|
|
124
183
|
print(json.dumps(result, ensure_ascii=False))
|
|
125
184
|
|
|
126
185
|
|
package/tools/report_renderer.py
CHANGED
|
@@ -12,7 +12,7 @@ from datetime import datetime, timezone
|
|
|
12
12
|
from pathlib import Path
|
|
13
13
|
|
|
14
14
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
15
|
-
from _subproc import utf8_stdio
|
|
15
|
+
from _subproc import error_message, utf8_stdio
|
|
16
16
|
from market_review import save_review
|
|
17
17
|
|
|
18
18
|
try:
|
|
@@ -55,12 +55,12 @@ def render(kind: str, template: str, report: dict) -> dict:
|
|
|
55
55
|
try:
|
|
56
56
|
tpl = _env().get_template(tpl_path)
|
|
57
57
|
except Exception as e:
|
|
58
|
-
return {"error": f"failed to load template {tpl_path}: {e}"}
|
|
58
|
+
return {"error": f"failed to load template {tpl_path}: {error_message(e)}"}
|
|
59
59
|
|
|
60
60
|
try:
|
|
61
61
|
content = tpl.render(**report)
|
|
62
62
|
except Exception as e:
|
|
63
|
-
return {"error": f"render failed: {e}", "template": tpl_path}
|
|
63
|
+
return {"error": f"render failed: {error_message(e)}", "template": tpl_path}
|
|
64
64
|
|
|
65
65
|
return {
|
|
66
66
|
"meta": {
|
package/tools/requirements.txt
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
akshare>=1.20.24; python_version >= "3.11"
|
|
2
2
|
akshare>=1.14.0; python_version < "3.11"
|
|
3
3
|
yfinance>=0.2.36
|
|
4
|
-
pandas>=2.0.0
|
|
4
|
+
pandas>=2.0.0,<4
|
|
5
5
|
numpy>=1.24.0
|
|
6
6
|
pyyaml>=6.0
|
|
7
7
|
pypinyin>=0.50.0
|
|
8
|
+
pypdf>=5.0.0
|
|
8
9
|
requests>=2.33.0
|
|
9
10
|
exchange-calendars>=4.5.0
|
|
10
|
-
|
|
11
|
+
jinja2>=3.1.6
|
|
12
|
+
# optional providers: failover-only sources; setup-python.mjs only warns when one cannot install
|
|
11
13
|
efinance>=0.5.0
|
|
12
14
|
baostock>=0.8.8
|
|
13
|
-
pytdx>=1.72
|
|
14
15
|
longport>=1.0.0
|
|
15
16
|
finnhub-python>=2.4.0
|
|
16
|
-
|
|
17
|
+
newspaper4k>=0.9.6.1
|
package/tools/risk_screening.py
CHANGED
|
@@ -7,12 +7,13 @@ import math
|
|
|
7
7
|
import os
|
|
8
8
|
import sys
|
|
9
9
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
10
|
-
from datetime import
|
|
10
|
+
from datetime import timedelta
|
|
11
11
|
|
|
12
12
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
13
13
|
|
|
14
14
|
from _subproc import run_tool as _run_tool
|
|
15
15
|
from _subproc import utf8_stdio
|
|
16
|
+
from trading_calendar import market_now
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
def _safe_float(val) -> float | None:
|
|
@@ -181,7 +182,7 @@ def _fetch_lockup(symbol: str) -> dict:
|
|
|
181
182
|
if df.empty:
|
|
182
183
|
return result
|
|
183
184
|
|
|
184
|
-
now =
|
|
185
|
+
now = market_now("CN").replace(hour=0, minute=0, second=0, microsecond=0)
|
|
185
186
|
cutoff = now + timedelta(days=30)
|
|
186
187
|
|
|
187
188
|
date_col = None
|
package/tools/screener.py
CHANGED
|
@@ -148,12 +148,13 @@ def compute_scores(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
148
148
|
mom_parts.append(_minmax(df["volume_ratio"]))
|
|
149
149
|
df["momentum_score"] = pd.concat(mom_parts, axis=1).mean(axis=1) if mom_parts else np.nan
|
|
150
150
|
|
|
151
|
-
# Liquidity: higher turnover_rate and
|
|
151
|
+
# Liquidity: higher turnover_rate and money traded → higher score. Share volume would
|
|
152
|
+
# favour low-priced stocks, so the cross-stock leg ranks turnover (成交额).
|
|
152
153
|
liq_parts = []
|
|
153
154
|
if "turnover_rate" in df.columns:
|
|
154
155
|
liq_parts.append(_minmax(df["turnover_rate"]))
|
|
155
|
-
if "
|
|
156
|
-
liq_parts.append(_minmax(df["
|
|
156
|
+
if "turnover" in df.columns:
|
|
157
|
+
liq_parts.append(_minmax(df["turnover"].astype(float)))
|
|
157
158
|
df["liquidity_score"] = pd.concat(liq_parts, axis=1).mean(axis=1) if liq_parts else np.nan
|
|
158
159
|
|
|
159
160
|
groups = df[[f"{group}_score" for group in L1_WEIGHTS]].rename(columns=lambda name: name.removesuffix("_score"))
|
|
@@ -465,6 +466,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
465
466
|
"price",
|
|
466
467
|
"change_pct",
|
|
467
468
|
"volume",
|
|
469
|
+
"turnover",
|
|
468
470
|
"turnover_rate",
|
|
469
471
|
"pe",
|
|
470
472
|
"pe_basis",
|
|
@@ -489,7 +491,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
489
491
|
factor_inputs = {
|
|
490
492
|
"value": ("pe", "pb", "dividend_yield"),
|
|
491
493
|
"momentum": ("change_pct", "volume_ratio"),
|
|
492
|
-
"liquidity": ("turnover_rate", "
|
|
494
|
+
"liquidity": ("turnover_rate", "turnover"),
|
|
493
495
|
}
|
|
494
496
|
weights = L1_WEIGHTS
|
|
495
497
|
available = [
|
package/tools/search_intel.py
CHANGED
|
@@ -9,9 +9,10 @@ import re
|
|
|
9
9
|
import sys
|
|
10
10
|
import time as _time
|
|
11
11
|
from contextvars import ContextVar
|
|
12
|
+
from urllib.parse import urlparse
|
|
12
13
|
|
|
13
14
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
14
|
-
from _subproc import scrub_secrets, utf8_stdio
|
|
15
|
+
from _subproc import error_message, scrub_secrets, utf8_stdio
|
|
15
16
|
|
|
16
17
|
_SEARCH_STATE = ContextVar("search_budget", default=None)
|
|
17
18
|
|
|
@@ -474,25 +475,90 @@ def get_trending_sentiment() -> dict:
|
|
|
474
475
|
# --------------- Article Extraction ---------------
|
|
475
476
|
|
|
476
477
|
|
|
477
|
-
def
|
|
478
|
+
def _extract_pdf(url: str, max_chars: int, response) -> dict:
|
|
479
|
+
import io
|
|
480
|
+
|
|
481
|
+
content = bytearray()
|
|
482
|
+
try:
|
|
483
|
+
from pypdf import PdfReader
|
|
484
|
+
|
|
485
|
+
for chunk in response.iter_content(65536):
|
|
486
|
+
content.extend(chunk)
|
|
487
|
+
if len(content) > 25 * 1024 * 1024:
|
|
488
|
+
raise ValueError("PDF exceeds the 25 MiB download limit")
|
|
489
|
+
finally:
|
|
490
|
+
response.close()
|
|
491
|
+
if not content.startswith(b"%PDF-"):
|
|
492
|
+
raise ValueError("URL did not return a PDF document")
|
|
493
|
+
|
|
494
|
+
reader = PdfReader(io.BytesIO(content))
|
|
495
|
+
parts = []
|
|
496
|
+
for page in reader.pages:
|
|
497
|
+
parts.append(page.extract_text() or "")
|
|
498
|
+
if sum(map(len, parts)) > max_chars:
|
|
499
|
+
break
|
|
500
|
+
text = "\n".join(parts)
|
|
501
|
+
metadata = reader.metadata or {}
|
|
502
|
+
return {
|
|
503
|
+
"url": url,
|
|
504
|
+
"title": str(metadata.get("/Title") or ""),
|
|
505
|
+
"text": text[:max_chars],
|
|
506
|
+
"authors": [str(metadata["/Author"])] if metadata.get("/Author") else [],
|
|
507
|
+
"publish_date": str(metadata["/CreationDate"]) if metadata.get("/CreationDate") else None,
|
|
508
|
+
"top_image": None,
|
|
509
|
+
"format": "pdf",
|
|
510
|
+
"page_count": len(reader.pages),
|
|
511
|
+
"truncated": len(text) > max_chars,
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def extract_article(url: str, max_chars: int = 3000) -> dict:
|
|
516
|
+
if isinstance(max_chars, bool) or not isinstance(max_chars, int) or not 1 <= max_chars <= 200000:
|
|
517
|
+
return {"url": url, "error": "max_chars must be an integer between 1 and 200000"}
|
|
518
|
+
is_pdf = url.lower().split("?", 1)[0].endswith(".pdf")
|
|
478
519
|
try:
|
|
479
|
-
from newspaper import Article
|
|
520
|
+
from newspaper import Article, network
|
|
521
|
+
|
|
522
|
+
config = network.Configuration()
|
|
523
|
+
config.allow_binary_content = True
|
|
524
|
+
config.requests_params.update(timeout=30, stream=True)
|
|
525
|
+
host = (urlparse(url).hostname or "").lower()
|
|
526
|
+
if host == "sec.gov" or host.endswith(".sec.gov"):
|
|
527
|
+
from sec_edgar import identity
|
|
528
|
+
|
|
529
|
+
config.browser_user_agent = identity() # SEC answers 403 to requests without a declared contact
|
|
530
|
+
response = network.do_request(url, config)
|
|
531
|
+
response.raise_for_status()
|
|
532
|
+
content_type = response.headers.get("content-type", "").lower()
|
|
533
|
+
is_pdf = is_pdf or "application/pdf" in content_type
|
|
534
|
+
if is_pdf:
|
|
535
|
+
return _extract_pdf(url, max_chars, response)
|
|
536
|
+
# allow_binary_content is on for PDFs only; refuse other binaries before reading the whole body.
|
|
537
|
+
if content_type.startswith(("image/", "video/", "audio/", "font/")) or (
|
|
538
|
+
content_type.startswith("application/") and "json" not in content_type and "xml" not in content_type
|
|
539
|
+
):
|
|
540
|
+
response.close()
|
|
541
|
+
return {"url": url, "error": f"Unsupported binary content-type: {content_type}"}
|
|
480
542
|
|
|
481
543
|
a = Article(url, language="zh")
|
|
482
|
-
a.download()
|
|
544
|
+
a.download(input_html=network.get_html(url, config, response=response))
|
|
483
545
|
a.parse()
|
|
546
|
+
text = a.text or ""
|
|
484
547
|
return {
|
|
485
548
|
"url": url,
|
|
486
549
|
"title": a.title or "",
|
|
487
|
-
"text":
|
|
550
|
+
"text": text[:max_chars],
|
|
488
551
|
"authors": a.authors or [],
|
|
489
552
|
"publish_date": str(a.publish_date) if a.publish_date else None,
|
|
490
553
|
"top_image": a.top_image or None,
|
|
554
|
+
"format": "html",
|
|
555
|
+
"truncated": len(text) > max_chars,
|
|
491
556
|
}
|
|
492
|
-
except ImportError:
|
|
493
|
-
|
|
557
|
+
except ImportError as exc:
|
|
558
|
+
dependency = "pypdf" if exc.name == "pypdf" else "newspaper4k"
|
|
559
|
+
return {"url": url, "error": f"{dependency} not installed. pip install {dependency}"}
|
|
494
560
|
except Exception as e:
|
|
495
|
-
return {"url": url, "error":
|
|
561
|
+
return {"url": url, "error": error_message(e)}
|
|
496
562
|
|
|
497
563
|
|
|
498
564
|
# --------------- CLI Commands ---------------
|
|
@@ -515,7 +581,7 @@ def cmd_trending(args):
|
|
|
515
581
|
|
|
516
582
|
|
|
517
583
|
def cmd_extract(args):
|
|
518
|
-
return extract_article(args.url)
|
|
584
|
+
return extract_article(args.url, max_chars=args.max_chars)
|
|
519
585
|
|
|
520
586
|
|
|
521
587
|
def main():
|
|
@@ -536,7 +602,8 @@ def main():
|
|
|
536
602
|
sub.add_parser("trending")
|
|
537
603
|
|
|
538
604
|
p_ext = sub.add_parser("extract")
|
|
539
|
-
p_ext.add_argument("url", help="Article URL to extract")
|
|
605
|
+
p_ext.add_argument("url", help="Article or PDF URL to extract")
|
|
606
|
+
p_ext.add_argument("--max-chars", type=int, default=3000, help="Maximum extracted characters (1-200000)")
|
|
540
607
|
|
|
541
608
|
args = parser.parse_args()
|
|
542
609
|
if not args.command:
|
|
@@ -552,6 +619,8 @@ def main():
|
|
|
552
619
|
}
|
|
553
620
|
result = dispatch[args.command](args)
|
|
554
621
|
print(json.dumps(result, ensure_ascii=False, default=str))
|
|
622
|
+
if isinstance(result, dict) and "error" in result:
|
|
623
|
+
sys.exit(1)
|
|
555
624
|
|
|
556
625
|
|
|
557
626
|
if __name__ == "__main__":
|
package/tools/sec_edgar.py
CHANGED
|
@@ -282,6 +282,272 @@ def form4(cik, filing):
|
|
|
282
282
|
return rows
|
|
283
283
|
|
|
284
284
|
|
|
285
|
+
# Announcement categories mapped to SEC form types; categories without an SEC equivalent are rejected.
|
|
286
|
+
FILING_FORMS = {
|
|
287
|
+
"annual": {"10-K", "10-K/A", "20-F", "20-F/A", "40-F", "40-F/A"},
|
|
288
|
+
"quarterly": {"10-Q", "10-Q/A"},
|
|
289
|
+
"current": {"8-K", "8-K/A", "6-K", "6-K/A"},
|
|
290
|
+
"issuance": {"S-1", "S-1/A", "S-3", "S-3/A", "F-1", "F-1/A", "F-3", "F-3/A", "424B2", "424B3", "424B4", "424B5"},
|
|
291
|
+
"reduction": {"144"},
|
|
292
|
+
"inquiry": {"UPLOAD", "CORRESP"},
|
|
293
|
+
}
|
|
294
|
+
FILING_FORMS["all"] = set().union(*FILING_FORMS.values()) | {"DEF 14A", "DEFA14A"}
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
DCF_CONCEPTS = {
|
|
298
|
+
"revenue": ("Revenues", "RevenueFromContractWithCustomerExcludingAssessedTax", "SalesRevenueNet"),
|
|
299
|
+
"operating_income": ("OperatingIncomeLoss",),
|
|
300
|
+
"pretax_income": (
|
|
301
|
+
"IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest",
|
|
302
|
+
"IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments",
|
|
303
|
+
),
|
|
304
|
+
"income_tax": ("IncomeTaxExpenseBenefit",),
|
|
305
|
+
"operating_cash_flow": ("NetCashProvidedByUsedInOperatingActivities",),
|
|
306
|
+
"capital_expenditure": ("PaymentsToAcquirePropertyPlantAndEquipment",),
|
|
307
|
+
"depreciation_amortization": ("DepreciationDepletionAndAmortization", "DepreciationAndAmortization"),
|
|
308
|
+
"diluted_shares": ("WeightedAverageNumberOfDilutedSharesOutstanding",),
|
|
309
|
+
}
|
|
310
|
+
DCF_INSTANTS = {
|
|
311
|
+
"cash": ("CashAndCashEquivalentsAtCarryingValue",),
|
|
312
|
+
"minority_interest": ("MinorityInterest",),
|
|
313
|
+
}
|
|
314
|
+
DEBT_CONCEPTS = (
|
|
315
|
+
"LongTermDebt",
|
|
316
|
+
"LongTermDebtCurrent",
|
|
317
|
+
"LongTermDebtNoncurrent",
|
|
318
|
+
"ShortTermBorrowings",
|
|
319
|
+
"CommercialPaper",
|
|
320
|
+
)
|
|
321
|
+
# Each lease kind: the total tag, else its current + noncurrent parts.
|
|
322
|
+
LEASE_CONCEPTS = {
|
|
323
|
+
"OperatingLeaseLiability": ("OperatingLeaseLiabilityCurrent", "OperatingLeaseLiabilityNoncurrent"),
|
|
324
|
+
"FinanceLeaseLiability": ("FinanceLeaseLiabilityCurrent", "FinanceLeaseLiabilityNoncurrent"),
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def annual_fact(row):
|
|
329
|
+
if row.get("fp") != "FY" or row.get("form") not in {"10-K", "10-K/A", "20-F", "20-F/A", "40-F", "40-F/A"}:
|
|
330
|
+
return False
|
|
331
|
+
if "start" not in row:
|
|
332
|
+
return True
|
|
333
|
+
days = (date.fromisoformat(row["end"]) - date.fromisoformat(row["start"])).days
|
|
334
|
+
return 300 <= days <= 400
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def latest_facts(facts, names, unit):
|
|
338
|
+
values = {}
|
|
339
|
+
for name in names:
|
|
340
|
+
concept = {}
|
|
341
|
+
for row in facts.get(name, {}).get("units", {}).get(unit, []):
|
|
342
|
+
if annual_fact(row) and (row["end"] not in concept or row.get("filed", "") > concept[row["end"]]["filed"]):
|
|
343
|
+
concept[row["end"]] = row
|
|
344
|
+
values.update({end: row for end, row in concept.items() if end not in values})
|
|
345
|
+
return values
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def company_facts(symbol):
|
|
349
|
+
cik = resolve_cik(symbol)
|
|
350
|
+
return cached(
|
|
351
|
+
f"sec-companyfacts:{cik}", DAY, lambda: sec_get(f"https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json")
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def dcf_inputs(symbol, periods):
|
|
356
|
+
data = company_facts(symbol)
|
|
357
|
+
gaap = data.get("facts", {}).get("us-gaap", {})
|
|
358
|
+
rows = {
|
|
359
|
+
key: latest_facts(gaap, names, "shares" if key == "diluted_shares" else "USD")
|
|
360
|
+
for key, names in {**DCF_CONCEPTS, **DCF_INSTANTS}.items()
|
|
361
|
+
}
|
|
362
|
+
lease_names = [name for total, parts in LEASE_CONCEPTS.items() for name in (total, *parts)]
|
|
363
|
+
debt = {name: latest_facts(gaap, (name,), "USD") for name in (*DEBT_CONCEPTS, *lease_names)}
|
|
364
|
+
# Values keep the latest (restated) filing; labels and cover shares come from each period's original report.
|
|
365
|
+
originals = {}
|
|
366
|
+
for name in DCF_CONCEPTS["revenue"] + DCF_CONCEPTS["operating_income"]:
|
|
367
|
+
for row in gaap.get(name, {}).get("units", {}).get("USD", []):
|
|
368
|
+
if annual_fact(row) and (row["end"] not in originals or row["filed"] < originals[row["end"]]["filed"]):
|
|
369
|
+
originals[row["end"]] = row
|
|
370
|
+
ends = sorted(originals, reverse=True)[:periods]
|
|
371
|
+
outstanding = (
|
|
372
|
+
data.get("facts", {})
|
|
373
|
+
.get("dei", {})
|
|
374
|
+
.get("EntityCommonStockSharesOutstanding", {})
|
|
375
|
+
.get("units", {})
|
|
376
|
+
.get("shares", [])
|
|
377
|
+
)
|
|
378
|
+
result = []
|
|
379
|
+
for end in ends:
|
|
380
|
+
entry = {key: rows[key].get(end, {}).get("val") for key in rows}
|
|
381
|
+
source = originals[end]
|
|
382
|
+
if (date.fromisoformat(source["filed"]) - date.fromisoformat(end)).days > 365:
|
|
383
|
+
# Only a later year's comparative exists (e.g. pre-XBRL years); its labels would be wrong.
|
|
384
|
+
source = {}
|
|
385
|
+
parts = [debt["LongTermDebt"].get(end, {}).get("val")]
|
|
386
|
+
if parts[0] is None:
|
|
387
|
+
parts = [debt[name].get(end, {}).get("val") for name in ("LongTermDebtCurrent", "LongTermDebtNoncurrent")]
|
|
388
|
+
parts += [debt[name].get(end, {}).get("val") for name in ("ShortTermBorrowings", "CommercialPaper")]
|
|
389
|
+
leases = []
|
|
390
|
+
for total, split in LEASE_CONCEPTS.items():
|
|
391
|
+
names = (total,) if end in debt[total] else split
|
|
392
|
+
if all(end in debt[name] for name in names):
|
|
393
|
+
leases += [debt[name][end]["val"] for name in names]
|
|
394
|
+
# Multi-class issuers report one cover row per class in the same filing; sum them.
|
|
395
|
+
cover = [
|
|
396
|
+
row
|
|
397
|
+
for row in outstanding
|
|
398
|
+
if source and (row.get("accn"), row.get("filed")) == (source.get("accn"), source.get("filed"))
|
|
399
|
+
]
|
|
400
|
+
entry.update(
|
|
401
|
+
{
|
|
402
|
+
"period_end": end,
|
|
403
|
+
"currency": "USD",
|
|
404
|
+
"total_debt": sum(value for value in parts if value is not None)
|
|
405
|
+
if any(value is not None for value in parts)
|
|
406
|
+
else None,
|
|
407
|
+
"lease_liabilities": sum(leases) if leases else None,
|
|
408
|
+
"shares_outstanding": sum(row["val"] for row in cover) if cover else None,
|
|
409
|
+
"shares_outstanding_date": max(row["end"] for row in cover) if cover else None,
|
|
410
|
+
"form": source.get("form"),
|
|
411
|
+
"filed": source.get("filed"),
|
|
412
|
+
"fiscal_year": source.get("fy"),
|
|
413
|
+
}
|
|
414
|
+
)
|
|
415
|
+
result.append(entry)
|
|
416
|
+
if not result:
|
|
417
|
+
raise ValueError("SEC annual DCF facts unavailable")
|
|
418
|
+
return {
|
|
419
|
+
"source": "sec_companyfacts",
|
|
420
|
+
"name": data.get("entityName"),
|
|
421
|
+
"periods": result,
|
|
422
|
+
"warnings": [
|
|
423
|
+
"SEC XBRL tags vary by issuer; null fields are filled from Yahoo for the same period when available. "
|
|
424
|
+
"total_debt sums reported debt/borrowing tags and excludes leases; lease_liabilities is operating + finance "
|
|
425
|
+
"leases. Amounts use the latest filed (possibly restated) value; form/filed/shares_outstanding come from "
|
|
426
|
+
"the period's original annual report."
|
|
427
|
+
],
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def first_filed(rows, forms=("10-Q", "10-Q/A", "10-K", "10-K/A")):
|
|
432
|
+
"""Earliest filing per (start, end): the value the market could see then, not a later restatement."""
|
|
433
|
+
kept = {}
|
|
434
|
+
for row in rows:
|
|
435
|
+
key = (row.get("start"), row["end"])
|
|
436
|
+
if row.get("form") in forms and row.get("filed") and (key not in kept or row["filed"] < kept[key]["filed"]):
|
|
437
|
+
kept[key] = row
|
|
438
|
+
return list(kept.values())
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def valuation_facts(symbol):
|
|
442
|
+
"""Point-in-time TTM net income, equity and share count events, each dated by its SEC filing date."""
|
|
443
|
+
data = company_facts(symbol)
|
|
444
|
+
gaap = data.get("facts", {}).get("us-gaap", {})
|
|
445
|
+
income = first_filed(gaap.get("NetIncomeLoss", {}).get("units", {}).get("USD", []))
|
|
446
|
+
|
|
447
|
+
def days(row):
|
|
448
|
+
return (date.fromisoformat(row["end"]) - date.fromisoformat(row["start"])).days
|
|
449
|
+
|
|
450
|
+
quarters = {row["end"]: row for row in income if "start" in row and 80 <= days(row) <= 100}
|
|
451
|
+
for year in (row for row in income if "start" in row and 350 <= days(row) <= 380):
|
|
452
|
+
inside = [q for q in quarters.values() if year["start"] <= q["start"] and q["end"] <= year["end"]]
|
|
453
|
+
if year["end"] not in quarters and len(inside) == 3:
|
|
454
|
+
quarters[year["end"]] = {
|
|
455
|
+
"start": max(q["end"] for q in inside),
|
|
456
|
+
"end": year["end"],
|
|
457
|
+
"val": year["val"] - sum(q["val"] for q in inside),
|
|
458
|
+
"filed": year["filed"],
|
|
459
|
+
}
|
|
460
|
+
ordered = sorted(quarters.values(), key=lambda row: row["end"])
|
|
461
|
+
ttm = []
|
|
462
|
+
for index in range(3, len(ordered)):
|
|
463
|
+
window = ordered[index - 3 : index + 1]
|
|
464
|
+
span = (date.fromisoformat(window[-1]["end"]) - date.fromisoformat(window[0]["start"])).days
|
|
465
|
+
if 350 <= span <= 380:
|
|
466
|
+
ttm.append(
|
|
467
|
+
{
|
|
468
|
+
"filed": max(q["filed"] for q in window),
|
|
469
|
+
"end": window[-1]["end"],
|
|
470
|
+
"val": sum(q["val"] for q in window),
|
|
471
|
+
}
|
|
472
|
+
)
|
|
473
|
+
equity = [
|
|
474
|
+
{"filed": row["filed"], "end": row["end"], "val": row["val"]}
|
|
475
|
+
for row in first_filed(gaap.get("StockholdersEquity", {}).get("units", {}).get("USD", []))
|
|
476
|
+
]
|
|
477
|
+
shares = {}
|
|
478
|
+
for row in (
|
|
479
|
+
data.get("facts", {})
|
|
480
|
+
.get("dei", {})
|
|
481
|
+
.get("EntityCommonStockSharesOutstanding", {})
|
|
482
|
+
.get("units", {})
|
|
483
|
+
.get("shares", [])
|
|
484
|
+
):
|
|
485
|
+
if row.get("form") in ("10-Q", "10-Q/A", "10-K", "10-K/A") and row.get("filed"):
|
|
486
|
+
# Multi-class issuers report one row per class under the same filing; sum them.
|
|
487
|
+
entry = shares.setdefault(
|
|
488
|
+
(row.get("accn"), row["end"]), {"filed": row["filed"], "end": row["end"], "val": 0}
|
|
489
|
+
)
|
|
490
|
+
entry["val"] += row["val"]
|
|
491
|
+
basis = "cover_shares_outstanding"
|
|
492
|
+
if not shares:
|
|
493
|
+
# Some multi-class filers (e.g. Alphabet) omit dei cover shares; weighted diluted shares are the fallback.
|
|
494
|
+
basis = "weighted_diluted_shares"
|
|
495
|
+
diluted = gaap.get("WeightedAverageNumberOfDilutedSharesOutstanding", {}).get("units", {}).get("shares", [])
|
|
496
|
+
shares = {row["end"]: row for row in first_filed(diluted) if "start" in row and 80 <= days(row) <= 100}
|
|
497
|
+
if not ttm or not equity or not shares:
|
|
498
|
+
raise ValueError("SEC net income, equity or share count facts unavailable")
|
|
499
|
+
return {
|
|
500
|
+
"name": data.get("entityName"),
|
|
501
|
+
"ttm_net_income": ttm,
|
|
502
|
+
"equity": equity,
|
|
503
|
+
"shares": list(shares.values()),
|
|
504
|
+
"share_basis": basis,
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def filings(symbol, category, start, end, top):
|
|
509
|
+
forms = FILING_FORMS.get(category)
|
|
510
|
+
if forms is None:
|
|
511
|
+
raise ValueError(
|
|
512
|
+
f"category {category} has no SEC form equivalent; use all/annual/quarterly/issuance/reduction/inquiry"
|
|
513
|
+
)
|
|
514
|
+
identity()
|
|
515
|
+
cik = resolve_cik(symbol)
|
|
516
|
+
_, rows, incomplete = submissions(cik, forms, start.isoformat())
|
|
517
|
+
rows = [r for r in rows if r.get("filingDate", "") <= end.isoformat()]
|
|
518
|
+
items = []
|
|
519
|
+
for row in rows[:top]:
|
|
520
|
+
base = archive_base(cik, row)
|
|
521
|
+
document = row.get("primaryDocument") or ""
|
|
522
|
+
form = row["form"]
|
|
523
|
+
items.append(
|
|
524
|
+
{
|
|
525
|
+
"title": f"{form} {row.get('primaryDocDescription') or ''}".strip(),
|
|
526
|
+
"date": row.get("filingDate"),
|
|
527
|
+
"report_date": row.get("reportDate") or None,
|
|
528
|
+
"form": form,
|
|
529
|
+
"category": next((k for k, v in FILING_FORMS.items() if k != "all" and form in v), "other"),
|
|
530
|
+
"accession": row["accessionNumber"],
|
|
531
|
+
"document_url": f"{base}/{document}" if re.fullmatch(r"[\w.-]+", document) else None,
|
|
532
|
+
"filing_url": f"{base}/{row['accessionNumber']}-index.html",
|
|
533
|
+
"pdf_url": None,
|
|
534
|
+
}
|
|
535
|
+
)
|
|
536
|
+
return {
|
|
537
|
+
"symbol": symbol,
|
|
538
|
+
"cik": cik,
|
|
539
|
+
"market": "US",
|
|
540
|
+
"source": "sec_edgar",
|
|
541
|
+
"start_date": start.isoformat(),
|
|
542
|
+
"end_date": end.isoformat(),
|
|
543
|
+
"total": len(rows),
|
|
544
|
+
"announcements": items,
|
|
545
|
+
"truncated": incomplete or len(rows) > top,
|
|
546
|
+
"body_extracted": False,
|
|
547
|
+
"requested_category": category,
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
|
|
285
551
|
def insider_trades(args):
|
|
286
552
|
identity()
|
|
287
553
|
if not 1 <= args.days <= 365 or not 1 <= args.top <= 20:
|
package/tools/sec_holders.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Exact issuer CUSIPs joined to the latest published SEC 13F bulk window."""
|
|
2
2
|
|
|
3
|
+
import contextlib
|
|
3
4
|
import csv
|
|
4
5
|
import hashlib
|
|
5
6
|
import io
|
|
@@ -129,6 +130,11 @@ def bulk_file(window):
|
|
|
129
130
|
raise ValueError("SEC bulk download is not a valid ZIP archive")
|
|
130
131
|
finally:
|
|
131
132
|
response.close()
|
|
133
|
+
# Each quarter is a ~100 MB archive; keep only the current window.
|
|
134
|
+
for old in path.parent.glob("sec-13f-*.zip"):
|
|
135
|
+
if old != path:
|
|
136
|
+
with contextlib.suppress(OSError):
|
|
137
|
+
old.unlink()
|
|
132
138
|
return path
|
|
133
139
|
except OSError as exc:
|
|
134
140
|
raise ValueError(f"SEC bulk archive unavailable ({type(exc).__name__})") from None
|