@weaxs/openclaw-stock-analysis 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.js +322 -21
- package/index.ts +257 -16
- package/openclaw.plugin.json +11 -1
- package/package.json +1 -1
- package/schemas/market_review_schema.json +12 -5
- package/schemas/report_schema.json +68 -4
- package/scripts/cli-timeout.mjs +27 -0
- package/scripts/json-input.mjs +26 -0
- package/scripts/setup-python.mjs +6 -6
- package/skills/bull-trend/SKILL.md +34 -10
- package/skills/bull-trend/references/fat-tails.md +91 -0
- package/skills/bull-trend/scripts/tail_analysis.py +132 -0
- package/skills/event-driven/SKILL.md +6 -0
- package/skills/growth-quality/SKILL.md +6 -0
- package/skills/market-review/SKILL.md +2 -0
- package/skills/stock-analysis/SKILL.md +25 -5
- package/skills/stock-analysis/references/stateless-analysis.md +66 -0
- package/skills/stock-screener/SKILL.md +7 -1
- package/skills/strategy-backtest/SKILL.md +13 -0
- package/skills/wisburg-research/SKILL.md +2 -0
- package/templates/market_review/full.md.j2 +7 -0
- package/templates/stock_analysis/_evidence.md.j2 +11 -0
- package/templates/stock_analysis/brief.md.j2 +6 -1
- package/templates/stock_analysis/full.md.j2 +8 -3
- package/tools/_subproc.py +66 -11
- package/tools/alert_rules.py +32 -4
- package/tools/backtest.py +95 -18
- package/tools/capabilities.py +11 -0
- package/tools/cli_budgets.json +26 -0
- package/tools/gather.py +24 -16
- package/tools/lixinger.py +100 -0
- package/tools/market_regime.py +7 -3
- package/tools/market_review.py +37 -15
- package/tools/position_context.py +20 -2
- package/tools/report_renderer.py +2 -0
- package/tools/risk_screening.py +45 -17
- package/tools/screener.py +74 -24
- package/tools/sec_edgar.py +346 -0
- package/tools/sec_holders.py +263 -0
- package/tools/signal_tracker.py +218 -24
- package/tools/stateless_analysis.py +630 -0
- package/tools/stock_data.py +170 -32
- package/tools/technical.py +16 -2
- package/tools/value_data.py +159 -0
- package/tools/watchlist.py +24 -1
- package/tools/watchlist_context.py +25 -10
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""Exact issuer CUSIPs joined to the latest published SEC 13F bulk window."""
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import hashlib
|
|
5
|
+
import io
|
|
6
|
+
import math
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import tempfile
|
|
10
|
+
import time
|
|
11
|
+
import zipfile
|
|
12
|
+
from collections import Counter
|
|
13
|
+
from datetime import date
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from xml.etree import ElementTree
|
|
16
|
+
|
|
17
|
+
import _cache
|
|
18
|
+
from sec_edgar import archive_base, sec_get, xml_root
|
|
19
|
+
|
|
20
|
+
CATALOG = "https://www.sec.gov/data-research/sec-markets-data/form-13f-data-sets"
|
|
21
|
+
MONTHS = ("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def sec_date(value):
|
|
25
|
+
value = value.strip()
|
|
26
|
+
if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
|
27
|
+
return date.fromisoformat(value).isoformat()
|
|
28
|
+
day, month, year = value.lower().split("-")
|
|
29
|
+
return date(int(year), MONTHS.index(month) + 1, int(day)).isoformat()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def issuer_cusips(cik, data):
|
|
33
|
+
recent = data["filings"]["recent"]
|
|
34
|
+
forms = recent.get("form", [])
|
|
35
|
+
candidates = [
|
|
36
|
+
i
|
|
37
|
+
for i, form in enumerate(forms)
|
|
38
|
+
if form
|
|
39
|
+
in (
|
|
40
|
+
"SCHEDULE 13G",
|
|
41
|
+
"SCHEDULE 13G/A",
|
|
42
|
+
"SCHEDULE 13D",
|
|
43
|
+
"SCHEDULE 13D/A",
|
|
44
|
+
"SC 13G",
|
|
45
|
+
"SC 13G/A",
|
|
46
|
+
"SC 13D",
|
|
47
|
+
"SC 13D/A",
|
|
48
|
+
)
|
|
49
|
+
]
|
|
50
|
+
evidence = []
|
|
51
|
+
for i in candidates[:10]:
|
|
52
|
+
document = recent["primaryDocument"][i].rsplit("/", 1)[-1]
|
|
53
|
+
if not re.fullmatch(r"[\w.-]+\.xml", document, re.I):
|
|
54
|
+
continue
|
|
55
|
+
filing = {"accessionNumber": recent["accessionNumber"][i]}
|
|
56
|
+
url = f"{archive_base(cik, filing)}/{document}"
|
|
57
|
+
try:
|
|
58
|
+
root = xml_root(sec_get(url, as_json=False))
|
|
59
|
+
except (ValueError, ElementTree.ParseError):
|
|
60
|
+
continue
|
|
61
|
+
issuer = root.find(".//issuerInfo")
|
|
62
|
+
if issuer is None:
|
|
63
|
+
continue
|
|
64
|
+
identifier = (issuer.findtext("issuerCik") or issuer.findtext("issuerCIK") or "").strip()
|
|
65
|
+
if not identifier.isdigit() or int(identifier) != int(cik):
|
|
66
|
+
continue
|
|
67
|
+
for node in issuer.findall(".//issuerCusipNumber"):
|
|
68
|
+
cusip = (node.text or "").strip().upper()
|
|
69
|
+
if re.fullmatch(r"[A-Z0-9*@#]{8}\d", cusip):
|
|
70
|
+
evidence.append(
|
|
71
|
+
{
|
|
72
|
+
"cusip": cusip,
|
|
73
|
+
"source_url": url,
|
|
74
|
+
"filing_date": recent["filingDate"][i],
|
|
75
|
+
"security_class": root.findtext(".//securitiesClassTitle"),
|
|
76
|
+
}
|
|
77
|
+
)
|
|
78
|
+
if not evidence:
|
|
79
|
+
raise ValueError("No verified issuer CIK/CUSIP mapping in recent structured SEC Schedule 13D/G filings")
|
|
80
|
+
return evidence
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def latest_bulk():
|
|
84
|
+
html = sec_get(CATALOG, as_json=False)
|
|
85
|
+
windows = []
|
|
86
|
+
for href in re.findall(r'href=["\']([^"\']+\.zip)["\']', html, re.I):
|
|
87
|
+
match = re.search(r"(\d{2}[a-z]{3}\d{4})-(\d{2}[a-z]{3}\d{4})_form13f\.zip$", href, re.I)
|
|
88
|
+
if not match:
|
|
89
|
+
continue
|
|
90
|
+
|
|
91
|
+
start, end = (sec_date(f"{value[:2]}-{value[2:5]}-{value[5:]}") for value in match.groups())
|
|
92
|
+
if href.startswith("/files/"):
|
|
93
|
+
url = "https://www.sec.gov" + href
|
|
94
|
+
elif href.startswith("https://www.sec.gov/files/"):
|
|
95
|
+
url = href
|
|
96
|
+
else:
|
|
97
|
+
continue
|
|
98
|
+
if end <= date.today().isoformat():
|
|
99
|
+
windows.append({"url": url, "start": start, "end": end})
|
|
100
|
+
if not windows:
|
|
101
|
+
raise ValueError("SEC 13F bulk catalog returned no published data window")
|
|
102
|
+
return max(windows, key=lambda w: w["end"])
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def bulk_file(window):
|
|
106
|
+
path = _cache.CACHE_DIR / f"sec-13f-{hashlib.sha256(window['url'].encode()).hexdigest()[:16]}.zip"
|
|
107
|
+
if path.exists() and 0 <= time.time() - path.stat().st_mtime < _cache.WEEK and zipfile.is_zipfile(path):
|
|
108
|
+
return path
|
|
109
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
110
|
+
tmp = None
|
|
111
|
+
try:
|
|
112
|
+
response = sec_get(window["url"], stream=True)
|
|
113
|
+
try:
|
|
114
|
+
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as out:
|
|
115
|
+
tmp = Path(out.name)
|
|
116
|
+
size = 0
|
|
117
|
+
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
|
118
|
+
size += len(chunk)
|
|
119
|
+
if size > 256 * 1024 * 1024:
|
|
120
|
+
raise ValueError("SEC bulk archive exceeds the 256 MiB download limit")
|
|
121
|
+
out.write(chunk)
|
|
122
|
+
finally:
|
|
123
|
+
response.close()
|
|
124
|
+
if not zipfile.is_zipfile(tmp):
|
|
125
|
+
raise ValueError("SEC bulk download is not a valid ZIP archive")
|
|
126
|
+
os.replace(tmp, path)
|
|
127
|
+
return path
|
|
128
|
+
except OSError as exc:
|
|
129
|
+
raise ValueError(f"SEC bulk archive unavailable ({type(exc).__name__})") from None
|
|
130
|
+
finally:
|
|
131
|
+
if tmp:
|
|
132
|
+
tmp.unlink(missing_ok=True)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def table_rows(archive, table):
|
|
136
|
+
names = [n for n in archive.namelist() if n.rsplit("/", 1)[-1].upper() == table + ".TSV"]
|
|
137
|
+
if len(names) != 1 or archive.getinfo(names[0]).file_size > 2 * 1024**3:
|
|
138
|
+
raise ValueError(f"SEC bulk {table} table missing or too large")
|
|
139
|
+
with archive.open(names[0]) as raw, io.TextIOWrapper(raw, encoding="utf-8-sig") as text:
|
|
140
|
+
reader = csv.DictReader(text, delimiter="\t")
|
|
141
|
+
if not reader.fieldnames or "ACCESSION_NUMBER" not in reader.fieldnames:
|
|
142
|
+
raise ValueError(f"SEC bulk {table} schema unavailable")
|
|
143
|
+
yield from reader
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def scan_holders(path, cusips):
|
|
147
|
+
with zipfile.ZipFile(path) as archive:
|
|
148
|
+
submissions = {r["ACCESSION_NUMBER"]: r for r in table_rows(archive, "SUBMISSION")}
|
|
149
|
+
covers = {r["ACCESSION_NUMBER"]: r for r in table_rows(archive, "COVERPAGE")}
|
|
150
|
+
periods = Counter(
|
|
151
|
+
sec_date(r["PERIODOFREPORT"]) for r in submissions.values() if r["SUBMISSIONTYPE"] == "13F-HR"
|
|
152
|
+
)
|
|
153
|
+
if not periods:
|
|
154
|
+
raise ValueError("SEC bulk contains no original 13F-HR reports")
|
|
155
|
+
# The modal report period is the complete quarterly cohort; late older reports occur in every archive.
|
|
156
|
+
period = max(periods, key=lambda p: (periods[p], p))
|
|
157
|
+
managers = {}
|
|
158
|
+
for accession, row in submissions.items():
|
|
159
|
+
if row["SUBMISSIONTYPE"] not in ("13F-HR", "13F-HR/A") or sec_date(row["PERIODOFREPORT"]) != period:
|
|
160
|
+
continue
|
|
161
|
+
row = {**row, "date": sec_date(row["FILING_DATE"]), "accession": accession, **covers.get(accession, {})}
|
|
162
|
+
managers.setdefault(row["CIK"].zfill(10), []).append(row)
|
|
163
|
+
by_accession = {row["accession"]: (cik, row) for cik, filings in managers.items() for row in filings}
|
|
164
|
+
selected = set()
|
|
165
|
+
incomplete = set()
|
|
166
|
+
for cik, filings in managers.items():
|
|
167
|
+
bases = [r for r in filings if r["SUBMISSIONTYPE"] == "13F-HR" or r.get("AMENDMENTTYPE") == "RESTATEMENT"]
|
|
168
|
+
base = max(bases, key=lambda r: (r["date"], r["accession"])) if bases else None
|
|
169
|
+
if base:
|
|
170
|
+
selected.add(base["accession"])
|
|
171
|
+
else:
|
|
172
|
+
incomplete.add(cik)
|
|
173
|
+
for r in filings:
|
|
174
|
+
if r["SUBMISSIONTYPE"] == "13F-HR/A" and (
|
|
175
|
+
not base or (r["date"], r["accession"]) > (base["date"], base["accession"])
|
|
176
|
+
):
|
|
177
|
+
if r.get("AMENDMENTTYPE") == "NEW HOLDINGS":
|
|
178
|
+
selected.add(r["accession"])
|
|
179
|
+
else:
|
|
180
|
+
incomplete.add(cik)
|
|
181
|
+
result = {}
|
|
182
|
+
# shortcut: stream one published archive per uncached CUSIP set; use a persistent DB if query volume warrants it.
|
|
183
|
+
for row in table_rows(archive, "INFOTABLE"):
|
|
184
|
+
accession = row["ACCESSION_NUMBER"]
|
|
185
|
+
if row["CUSIP"].strip().upper() not in cusips or accession not in by_accession:
|
|
186
|
+
continue
|
|
187
|
+
cik, filing = by_accession[accession]
|
|
188
|
+
if accession not in selected and cik not in incomplete:
|
|
189
|
+
continue
|
|
190
|
+
holder = result.setdefault(
|
|
191
|
+
cik,
|
|
192
|
+
{
|
|
193
|
+
"cik": cik,
|
|
194
|
+
"name": filing.get("FILINGMANAGER_NAME"),
|
|
195
|
+
"positions": [],
|
|
196
|
+
"holdings_complete": cik not in incomplete,
|
|
197
|
+
},
|
|
198
|
+
)
|
|
199
|
+
shares, value = float(row["SSHPRNAMT"]), float(row["VALUE"])
|
|
200
|
+
if not math.isfinite(shares) or not math.isfinite(value):
|
|
201
|
+
raise ValueError("SEC bulk contains nonfinite holding amounts")
|
|
202
|
+
multiplier = 1 if filing["date"] >= "2023-01-03" else 1000
|
|
203
|
+
holder["positions"].append(
|
|
204
|
+
{
|
|
205
|
+
"cusip": row["CUSIP"].strip().upper(),
|
|
206
|
+
"security_class": row.get("TITLEOFCLASS"),
|
|
207
|
+
"shares": shares,
|
|
208
|
+
"share_type": row["SSHPRNAMTTYPE"],
|
|
209
|
+
"put_call": row.get("PUTCALL") or None,
|
|
210
|
+
"reported_value_usd": value * multiplier,
|
|
211
|
+
"filing_date": filing["date"],
|
|
212
|
+
"accession": accession,
|
|
213
|
+
"form": filing["SUBMISSIONTYPE"],
|
|
214
|
+
"amendment_applied": accession in selected,
|
|
215
|
+
"filing_url": f"{archive_base(cik, {'accessionNumber': accession})}/{accession}-index.html",
|
|
216
|
+
}
|
|
217
|
+
)
|
|
218
|
+
for holder in result.values():
|
|
219
|
+
positions = holder["positions"]
|
|
220
|
+
holder["shares"] = (
|
|
221
|
+
sum(r["shares"] for r in positions if r["share_type"] == "SH" and not r["put_call"])
|
|
222
|
+
if holder["holdings_complete"]
|
|
223
|
+
else None
|
|
224
|
+
)
|
|
225
|
+
holder["reported_value_usd"] = (
|
|
226
|
+
sum(r["reported_value_usd"] for r in positions) if holder["holdings_complete"] else None
|
|
227
|
+
)
|
|
228
|
+
return (
|
|
229
|
+
period,
|
|
230
|
+
len(incomplete),
|
|
231
|
+
sorted(result.values(), key=lambda r: (-(r["reported_value_usd"] or 0), r["cik"])),
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def issuer_holders(cik, data):
|
|
236
|
+
evidence = issuer_cusips(cik, data)
|
|
237
|
+
cusips = sorted({r["cusip"] for r in evidence})
|
|
238
|
+
window = latest_bulk()
|
|
239
|
+
|
|
240
|
+
def fetch():
|
|
241
|
+
period, unresolved, holders = scan_holders(bulk_file(window), set(cusips))
|
|
242
|
+
return {"report_date": period, "institutions": holders, "unresolved_manager_count": unresolved}
|
|
243
|
+
|
|
244
|
+
result = _cache.cached(f"sec-issuer-holders:v2:{window['url']}:{','.join(cusips)}", _cache.DAY, fetch)
|
|
245
|
+
return {
|
|
246
|
+
"cik": cik,
|
|
247
|
+
"issuer": data.get("name"),
|
|
248
|
+
"query_type": "issuer",
|
|
249
|
+
"source": "sec_edgar",
|
|
250
|
+
"cusips": cusips,
|
|
251
|
+
"cusip_evidence": evidence,
|
|
252
|
+
**result,
|
|
253
|
+
"institution_count": len(result["institutions"]),
|
|
254
|
+
"bulk_source_url": window["url"],
|
|
255
|
+
"filing_window_start": window["start"],
|
|
256
|
+
"filing_window_end": window["end"],
|
|
257
|
+
"mapping_complete": False,
|
|
258
|
+
"complete_for_mapped_cusips": result["unresolved_manager_count"] == 0,
|
|
259
|
+
"realtime": False,
|
|
260
|
+
"value_unit": "USD",
|
|
261
|
+
"coverage": "Published SEC 13F window for verified CUSIPs; mapping may omit other security classes/CUSIP changes; late filings, confidential positions and subsequent amendments are not included",
|
|
262
|
+
"shares_basis": "SH equity positions only; options and principal amounts remain separate in positions",
|
|
263
|
+
}
|
package/tools/signal_tracker.py
CHANGED
|
@@ -10,15 +10,19 @@ host decides when to record and when to evaluate.
|
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
12
|
import argparse
|
|
13
|
+
import contextlib
|
|
13
14
|
import json
|
|
15
|
+
import math
|
|
14
16
|
import os
|
|
15
17
|
import sys
|
|
18
|
+
import tempfile
|
|
19
|
+
import time
|
|
16
20
|
from datetime import datetime
|
|
17
21
|
from pathlib import Path
|
|
18
22
|
from uuid import uuid4
|
|
19
23
|
|
|
20
24
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
21
|
-
from _subproc import run_tool, utf8_stdio # noqa: E402
|
|
25
|
+
from _subproc import command_timeout, run_tool, utf8_stdio # noqa: E402
|
|
22
26
|
from stock_data import detect_market # noqa: E402
|
|
23
27
|
|
|
24
28
|
|
|
@@ -31,26 +35,108 @@ def _store_path(args) -> Path:
|
|
|
31
35
|
return Path(args.store) if args.store else _default_store()
|
|
32
36
|
|
|
33
37
|
|
|
34
|
-
def
|
|
38
|
+
def _validate_signal(record):
|
|
39
|
+
if not isinstance(record, dict) or any(
|
|
40
|
+
not isinstance(record.get(field), str) or not record[field] for field in ("id", "symbol", "record_date")
|
|
41
|
+
):
|
|
42
|
+
raise ValueError("Invalid signal identity/date")
|
|
43
|
+
if record.get("status") not in ("open", "closed") or record.get("direction") not in ("buy", "sell"):
|
|
44
|
+
raise ValueError("Invalid signal status/direction")
|
|
45
|
+
if record.get("source") is not None and not isinstance(record["source"], str):
|
|
46
|
+
raise ValueError("Signal source must be a string")
|
|
47
|
+
if record.get("entry_price") is None or any(
|
|
48
|
+
isinstance(record.get(field), bool)
|
|
49
|
+
or not isinstance(record[field], (int, float))
|
|
50
|
+
or not math.isfinite(record[field])
|
|
51
|
+
or record[field] <= 0
|
|
52
|
+
for field in ("entry_price", "target_price", "stop_price")
|
|
53
|
+
if record.get(field) is not None
|
|
54
|
+
):
|
|
55
|
+
raise ValueError("Signal prices must be finite and positive")
|
|
56
|
+
horizon = record.get("horizon_days")
|
|
57
|
+
if isinstance(horizon, bool) or not isinstance(horizon, int) or horizon <= 0:
|
|
58
|
+
raise ValueError("Signal horizon_days must be a positive integer")
|
|
59
|
+
try:
|
|
60
|
+
date = datetime.strptime(record["record_date"], "%Y-%m-%d")
|
|
61
|
+
except ValueError as e:
|
|
62
|
+
raise ValueError("Signal date must be YYYY-MM-DD") from e
|
|
63
|
+
if date.strftime("%Y-%m-%d") != record["record_date"]:
|
|
64
|
+
raise ValueError("Signal date must be YYYY-MM-DD")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@contextlib.contextmanager
|
|
68
|
+
def _store_lock(store: Path):
|
|
69
|
+
# Lock a separate file: replacing the JSONL must not replace the lock inode.
|
|
70
|
+
store.parent.mkdir(parents=True, exist_ok=True)
|
|
71
|
+
with store.with_name(store.name + ".lock").open("a+b") as lock:
|
|
72
|
+
if sys.platform == "win32":
|
|
73
|
+
import msvcrt
|
|
74
|
+
|
|
75
|
+
if lock.seek(0, os.SEEK_END) == 0:
|
|
76
|
+
lock.write(b"\0")
|
|
77
|
+
lock.flush()
|
|
78
|
+
lock.seek(0)
|
|
79
|
+
msvcrt.locking(lock.fileno(), msvcrt.LK_LOCK, 1)
|
|
80
|
+
else:
|
|
81
|
+
import fcntl
|
|
82
|
+
|
|
83
|
+
fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
|
|
84
|
+
try:
|
|
85
|
+
yield
|
|
86
|
+
finally:
|
|
87
|
+
if sys.platform == "win32":
|
|
88
|
+
lock.seek(0)
|
|
89
|
+
msvcrt.locking(lock.fileno(), msvcrt.LK_UNLCK, 1)
|
|
90
|
+
else:
|
|
91
|
+
fcntl.flock(lock.fileno(), fcntl.LOCK_UN)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _load_records(store: Path, strict: bool = False) -> list:
|
|
35
95
|
if not store.exists():
|
|
36
96
|
return []
|
|
37
97
|
records = []
|
|
38
|
-
for line in store.read_text(encoding="utf-8").splitlines():
|
|
98
|
+
for number, line in enumerate(store.read_text(encoding="utf-8").splitlines(), 1):
|
|
39
99
|
line = line.strip()
|
|
40
100
|
if line:
|
|
41
101
|
try:
|
|
42
|
-
|
|
43
|
-
except json.JSONDecodeError:
|
|
102
|
+
record = json.loads(line)
|
|
103
|
+
except json.JSONDecodeError as e:
|
|
104
|
+
if strict:
|
|
105
|
+
raise ValueError(f"Invalid signal JSON at line {number}; store left unchanged") from e
|
|
44
106
|
continue
|
|
107
|
+
if strict:
|
|
108
|
+
try:
|
|
109
|
+
_validate_signal(record)
|
|
110
|
+
except ValueError as e:
|
|
111
|
+
raise ValueError(f"Invalid signal record at line {number}; store left unchanged: {e}") from e
|
|
112
|
+
records.append(record)
|
|
45
113
|
return records
|
|
46
114
|
|
|
47
115
|
|
|
48
116
|
def _save_records(store: Path, records: list):
|
|
49
|
-
"""Atomic rewrite
|
|
117
|
+
"""Atomic rewrite; caller holds the store lock."""
|
|
50
118
|
store.parent.mkdir(parents=True, exist_ok=True)
|
|
51
|
-
tmp =
|
|
52
|
-
|
|
53
|
-
|
|
119
|
+
tmp = None
|
|
120
|
+
try:
|
|
121
|
+
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=store.parent, delete=False) as stream:
|
|
122
|
+
tmp = Path(stream.name)
|
|
123
|
+
stream.write("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records))
|
|
124
|
+
os.replace(tmp, store)
|
|
125
|
+
finally:
|
|
126
|
+
if tmp:
|
|
127
|
+
with contextlib.suppress(OSError):
|
|
128
|
+
tmp.unlink(missing_ok=True)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _append_record(store: Path, record: dict):
|
|
132
|
+
"""Append one complete JSON line; caller holds the store lock."""
|
|
133
|
+
payload = (json.dumps(record, ensure_ascii=False, default=str, allow_nan=False) + "\n").encode("utf-8")
|
|
134
|
+
with store.open("a+b") as stream:
|
|
135
|
+
if stream.seek(0, os.SEEK_END):
|
|
136
|
+
stream.seek(-1, os.SEEK_END)
|
|
137
|
+
if stream.read(1) != b"\n":
|
|
138
|
+
stream.write(b"\n")
|
|
139
|
+
stream.write(payload)
|
|
54
140
|
|
|
55
141
|
|
|
56
142
|
def cmd_record(args):
|
|
@@ -78,10 +164,11 @@ def cmd_record(args):
|
|
|
78
164
|
}
|
|
79
165
|
store = _store_path(args)
|
|
80
166
|
try:
|
|
81
|
-
|
|
82
|
-
with store
|
|
83
|
-
|
|
84
|
-
|
|
167
|
+
_validate_signal(record)
|
|
168
|
+
with _store_lock(store):
|
|
169
|
+
_load_records(store, strict=True)
|
|
170
|
+
_append_record(store, record)
|
|
171
|
+
except (OSError, ValueError) as e:
|
|
85
172
|
return {"error": str(e)}
|
|
86
173
|
return record
|
|
87
174
|
|
|
@@ -135,31 +222,90 @@ def _kline_count(rec: dict) -> int:
|
|
|
135
222
|
return min(max(horizon + 40, elapsed + 15), 800)
|
|
136
223
|
|
|
137
224
|
|
|
225
|
+
def _valid_bar(bar) -> bool:
|
|
226
|
+
if not isinstance(bar, dict):
|
|
227
|
+
return False
|
|
228
|
+
try:
|
|
229
|
+
date = datetime.strptime(bar["date"], "%Y-%m-%d")
|
|
230
|
+
except (KeyError, TypeError, ValueError):
|
|
231
|
+
return False
|
|
232
|
+
return date.strftime("%Y-%m-%d") == bar["date"] and all(
|
|
233
|
+
isinstance(bar.get(key), (int, float)) and math.isfinite(bar[key]) and bar[key] > 0
|
|
234
|
+
for key in ("high", "low", "close")
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _price_factor(rec, bars):
|
|
239
|
+
if all(bar.get("adjusted") == "none" for bar in bars):
|
|
240
|
+
return 1.0
|
|
241
|
+
anchor = next((bar for bar in bars if bar["date"] == rec["record_date"]), {})
|
|
242
|
+
basis = anchor.get("adjusted")
|
|
243
|
+
factor = anchor.get("adjustment_factor")
|
|
244
|
+
if (
|
|
245
|
+
basis in ("qfq", "total_return")
|
|
246
|
+
and all(bar.get("adjusted") == basis for bar in bars)
|
|
247
|
+
and not isinstance(factor, bool)
|
|
248
|
+
and isinstance(factor, (int, float))
|
|
249
|
+
and math.isfinite(factor)
|
|
250
|
+
and factor > 0
|
|
251
|
+
):
|
|
252
|
+
return factor
|
|
253
|
+
return None
|
|
254
|
+
|
|
255
|
+
|
|
138
256
|
def cmd_evaluate(args):
|
|
257
|
+
deadline = time.monotonic() + command_timeout("signal_tracker.py", ["evaluate"]) - 30
|
|
139
258
|
store = _store_path(args)
|
|
140
|
-
|
|
259
|
+
try:
|
|
260
|
+
with _store_lock(store):
|
|
261
|
+
records = _load_records(store, strict=True)
|
|
262
|
+
except (OSError, ValueError) as e:
|
|
263
|
+
return {"error": str(e)}
|
|
141
264
|
results = []
|
|
142
265
|
evaluated = closed = 0
|
|
143
266
|
changed = False
|
|
267
|
+
deferred = 0
|
|
144
268
|
for rec in records:
|
|
145
269
|
if rec.get("status") != "open":
|
|
146
270
|
continue
|
|
147
271
|
if args.symbol and rec.get("symbol") != args.symbol:
|
|
148
272
|
continue
|
|
273
|
+
remaining = deadline - time.monotonic()
|
|
274
|
+
if remaining <= 0:
|
|
275
|
+
deferred += 1
|
|
276
|
+
continue
|
|
149
277
|
evaluated += 1
|
|
150
278
|
entry = {"id": rec.get("id"), "symbol": rec.get("symbol")}
|
|
151
279
|
bars = run_tool(
|
|
152
280
|
"stock_data.py",
|
|
153
281
|
["kline", rec["symbol"], "--period", "daily", "--count", str(_kline_count(rec))],
|
|
282
|
+
timeout=min(command_timeout("stock_data.py", ["kline"]), remaining),
|
|
154
283
|
)
|
|
155
|
-
if
|
|
284
|
+
if (
|
|
285
|
+
not isinstance(bars, list)
|
|
286
|
+
or not bars
|
|
287
|
+
or any(not _valid_bar(bar) for bar in bars)
|
|
288
|
+
or min(str(b.get("date", ""))[:10] for b in bars) > rec["record_date"]
|
|
289
|
+
):
|
|
156
290
|
entry.update({"outcome": "skipped_no_data", "return_pct": None, "exit_date": None})
|
|
291
|
+
elif (factor := _price_factor(rec, bars)) is None:
|
|
292
|
+
entry.update(
|
|
293
|
+
outcome="skipped_price_basis",
|
|
294
|
+
return_pct=None,
|
|
295
|
+
exit_date=None,
|
|
296
|
+
warning="Adjusted or unknown prices lack a verified conversion to the original signal prices",
|
|
297
|
+
)
|
|
157
298
|
else:
|
|
158
|
-
|
|
299
|
+
priced = dict(rec)
|
|
300
|
+
for key in ("entry_price", "target_price", "stop_price"):
|
|
301
|
+
if priced.get(key) is not None:
|
|
302
|
+
priced[key] *= factor
|
|
303
|
+
ev = _evaluate_bars(priced, bars)
|
|
159
304
|
if ev is None:
|
|
160
305
|
entry.update({"outcome": "open", "return_pct": None, "exit_date": None})
|
|
161
306
|
else:
|
|
162
307
|
outcome, exit_price, exit_date = ev
|
|
308
|
+
exit_price /= factor
|
|
163
309
|
ret = round((exit_price - rec["entry_price"]) / rec["entry_price"] * 100, 2)
|
|
164
310
|
if rec.get("direction") == "sell":
|
|
165
311
|
ret = -ret
|
|
@@ -170,22 +316,57 @@ def cmd_evaluate(args):
|
|
|
170
316
|
"exit_price": exit_price,
|
|
171
317
|
"exit_date": exit_date,
|
|
172
318
|
"return_pct": ret,
|
|
319
|
+
"evaluation_price_basis": {"adjusted": bars[0].get("adjusted"), "factor": factor},
|
|
173
320
|
}
|
|
174
321
|
)
|
|
175
|
-
closed += 1
|
|
176
322
|
changed = True
|
|
177
323
|
entry.update({"outcome": outcome, "return_pct": ret, "exit_date": exit_date})
|
|
178
324
|
results.append(entry)
|
|
179
|
-
if
|
|
325
|
+
if results:
|
|
180
326
|
try:
|
|
181
|
-
|
|
182
|
-
|
|
327
|
+
updates = {r["id"]: r for r in records if r.get("status") == "closed"}
|
|
328
|
+
with _store_lock(store):
|
|
329
|
+
current = _load_records(store, strict=True)
|
|
330
|
+
merged = [updates.get(r["id"], r) if r.get("status") == "open" else r for r in current]
|
|
331
|
+
if deferred:
|
|
332
|
+
attempted = {entry["id"] for entry in results}
|
|
333
|
+
positions = [
|
|
334
|
+
i
|
|
335
|
+
for i, rec in enumerate(merged)
|
|
336
|
+
if rec.get("status") == "open" and (not args.symbol or rec.get("symbol") == args.symbol)
|
|
337
|
+
]
|
|
338
|
+
# Keep closed/unselected records in place; give unattempted signals the next turn.
|
|
339
|
+
queued = sorted((merged[i] for i in positions), key=lambda rec: rec["id"] in attempted)
|
|
340
|
+
for i, rec in zip(positions, queued):
|
|
341
|
+
merged[i] = rec
|
|
342
|
+
if changed or deferred:
|
|
343
|
+
_save_records(store, merged)
|
|
344
|
+
persisted = {r["id"]: r for r in merged}
|
|
345
|
+
for entry in results:
|
|
346
|
+
rec = persisted[entry["id"]]
|
|
347
|
+
if rec.get("status") == "closed":
|
|
348
|
+
entry.update({key: rec.get(key) for key in ("outcome", "return_pct", "exit_date")})
|
|
349
|
+
closed += 1
|
|
350
|
+
except (OSError, ValueError) as e:
|
|
183
351
|
return {"error": str(e)}
|
|
184
|
-
return {
|
|
352
|
+
return {
|
|
353
|
+
"evaluated": evaluated,
|
|
354
|
+
"closed": closed,
|
|
355
|
+
"still_open": evaluated + deferred - closed,
|
|
356
|
+
"results": results,
|
|
357
|
+
"deferred": deferred,
|
|
358
|
+
"complete": deferred == 0,
|
|
359
|
+
"warnings": ["Time budget exhausted; deferred signals remain open for the next evaluation"] if deferred else [],
|
|
360
|
+
}
|
|
185
361
|
|
|
186
362
|
|
|
187
363
|
def cmd_summary(args):
|
|
188
|
-
|
|
364
|
+
store = _store_path(args)
|
|
365
|
+
try:
|
|
366
|
+
with _store_lock(store):
|
|
367
|
+
records = _load_records(store, strict=True)
|
|
368
|
+
except (OSError, ValueError) as e:
|
|
369
|
+
return {"error": str(e)}
|
|
189
370
|
filtered = records
|
|
190
371
|
if args.source:
|
|
191
372
|
filtered = [r for r in filtered if r.get("source") == args.source]
|
|
@@ -201,14 +382,27 @@ def cmd_summary(args):
|
|
|
201
382
|
for o in ("target_hit", "stop_hit", "timeout_profit", "timeout_loss")
|
|
202
383
|
}
|
|
203
384
|
wins = counts["target_hit"] + counts["timeout_profit"]
|
|
204
|
-
returns = [
|
|
385
|
+
returns = [
|
|
386
|
+
r["return_pct"]
|
|
387
|
+
for r in closed_recs
|
|
388
|
+
if isinstance(r.get("return_pct"), (int, float))
|
|
389
|
+
and not isinstance(r["return_pct"], bool)
|
|
390
|
+
and math.isfinite(r["return_pct"])
|
|
391
|
+
]
|
|
205
392
|
by_source = {}
|
|
206
393
|
for r in closed_recs:
|
|
207
|
-
agg = by_source.setdefault(r.get("source") or "unknown", {"closed": 0, "wins": 0})
|
|
394
|
+
agg = by_source.setdefault(r.get("source") or "unknown", {"closed": 0, "wins": 0, "returns": []})
|
|
208
395
|
agg["closed"] += 1
|
|
396
|
+
value = r.get("return_pct")
|
|
397
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value):
|
|
398
|
+
agg["returns"].append(value)
|
|
209
399
|
if r.get("outcome") in ("target_hit", "timeout_profit"):
|
|
210
400
|
agg["wins"] += 1
|
|
211
401
|
for agg in by_source.values():
|
|
402
|
+
samples = agg.pop("returns")
|
|
403
|
+
agg["return_samples"] = len(samples)
|
|
404
|
+
agg["avg_return_pct"] = round(sum(samples) / len(samples), 2) if samples else None
|
|
405
|
+
agg["sample_warning"] = "Small observational sample; not strategy validation" if len(samples) < 15 else None
|
|
212
406
|
agg["win_rate"] = round(agg["wins"] / agg["closed"] * 100, 2) if agg["closed"] else None
|
|
213
407
|
return {
|
|
214
408
|
"filters": {"source": args.source, "symbol": args.symbol, "status": args.status},
|