@zigrivers/scaffold 3.14.0 → 3.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -9
- package/content/knowledge/research/research-architecture.md +385 -0
- package/content/knowledge/research/research-conventions.md +248 -0
- package/content/knowledge/research/research-dev-environment.md +303 -0
- package/content/knowledge/research/research-experiment-loop.md +429 -0
- package/content/knowledge/research/research-experiment-tracking.md +336 -0
- package/content/knowledge/research/research-ml-architecture-search.md +383 -0
- package/content/knowledge/research/research-ml-evaluation.md +407 -0
- package/content/knowledge/research/research-ml-experiment-tracking.md +466 -0
- package/content/knowledge/research/research-ml-training-patterns.md +413 -0
- package/content/knowledge/research/research-observability.md +395 -0
- package/content/knowledge/research/research-overfitting-prevention.md +306 -0
- package/content/knowledge/research/research-project-structure.md +264 -0
- package/content/knowledge/research/research-quant-backtesting.md +326 -0
- package/content/knowledge/research/research-quant-market-data.md +366 -0
- package/content/knowledge/research/research-quant-metrics.md +335 -0
- package/content/knowledge/research/research-quant-requirements.md +223 -0
- package/content/knowledge/research/research-quant-risk.md +469 -0
- package/content/knowledge/research/research-quant-strategy-patterns.md +412 -0
- package/content/knowledge/research/research-requirements.md +201 -0
- package/content/knowledge/research/research-security.md +374 -0
- package/content/knowledge/research/research-sim-compute-management.md +538 -0
- package/content/knowledge/research/research-sim-engine-patterns.md +448 -0
- package/content/knowledge/research/research-sim-parameter-spaces.md +425 -0
- package/content/knowledge/research/research-sim-validation.md +456 -0
- package/content/knowledge/research/research-testing.md +334 -0
- package/content/methodology/research-ml-research.yml +23 -0
- package/content/methodology/research-overlay.yml +65 -0
- package/content/methodology/research-quant-finance.yml +29 -0
- package/content/methodology/research-simulation.yml +23 -0
- package/dist/cli/commands/adopt.d.ts.map +1 -1
- package/dist/cli/commands/adopt.js +22 -1
- package/dist/cli/commands/adopt.js.map +1 -1
- package/dist/cli/commands/adopt.serialization.test.js +41 -0
- package/dist/cli/commands/adopt.serialization.test.js.map +1 -1
- package/dist/cli/commands/init.d.ts +4 -0
- package/dist/cli/commands/init.d.ts.map +1 -1
- package/dist/cli/commands/init.js +32 -2
- package/dist/cli/commands/init.js.map +1 -1
- package/dist/cli/init-flag-families.d.ts +6 -1
- package/dist/cli/init-flag-families.d.ts.map +1 -1
- package/dist/cli/init-flag-families.js +32 -1
- package/dist/cli/init-flag-families.js.map +1 -1
- package/dist/cli/init-flag-families.test.js +47 -0
- package/dist/cli/init-flag-families.test.js.map +1 -1
- package/dist/config/schema.d.ts +272 -16
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +25 -1
- package/dist/config/schema.js.map +1 -1
- package/dist/config/schema.test.js +103 -3
- package/dist/config/schema.test.js.map +1 -1
- package/dist/core/assembly/overlay-loader.d.ts +12 -0
- package/dist/core/assembly/overlay-loader.d.ts.map +1 -1
- package/dist/core/assembly/overlay-loader.js +30 -0
- package/dist/core/assembly/overlay-loader.js.map +1 -1
- package/dist/core/assembly/overlay-loader.test.js +66 -1
- package/dist/core/assembly/overlay-loader.test.js.map +1 -1
- package/dist/core/assembly/overlay-state-resolver.d.ts.map +1 -1
- package/dist/core/assembly/overlay-state-resolver.js +48 -19
- package/dist/core/assembly/overlay-state-resolver.js.map +1 -1
- package/dist/core/assembly/overlay-state-resolver.test.js +80 -0
- package/dist/core/assembly/overlay-state-resolver.test.js.map +1 -1
- package/dist/e2e/project-type-overlays.test.js +119 -0
- package/dist/e2e/project-type-overlays.test.js.map +1 -1
- package/dist/project/adopt.d.ts.map +1 -1
- package/dist/project/adopt.js +3 -1
- package/dist/project/adopt.js.map +1 -1
- package/dist/project/detectors/disambiguate.js +1 -1
- package/dist/project/detectors/disambiguate.js.map +1 -1
- package/dist/project/detectors/index.d.ts.map +1 -1
- package/dist/project/detectors/index.js +2 -1
- package/dist/project/detectors/index.js.map +1 -1
- package/dist/project/detectors/ml.d.ts.map +1 -1
- package/dist/project/detectors/ml.js +2 -6
- package/dist/project/detectors/ml.js.map +1 -1
- package/dist/project/detectors/research.d.ts +4 -0
- package/dist/project/detectors/research.d.ts.map +1 -0
- package/dist/project/detectors/research.js +141 -0
- package/dist/project/detectors/research.js.map +1 -0
- package/dist/project/detectors/research.test.d.ts +2 -0
- package/dist/project/detectors/research.test.d.ts.map +1 -0
- package/dist/project/detectors/research.test.js +235 -0
- package/dist/project/detectors/research.test.js.map +1 -0
- package/dist/project/detectors/shared-signals.d.ts +3 -0
- package/dist/project/detectors/shared-signals.d.ts.map +1 -0
- package/dist/project/detectors/shared-signals.js +9 -0
- package/dist/project/detectors/shared-signals.js.map +1 -0
- package/dist/project/detectors/types.d.ts +6 -2
- package/dist/project/detectors/types.d.ts.map +1 -1
- package/dist/project/detectors/types.js.map +1 -1
- package/dist/types/config.d.ts +7 -1
- package/dist/types/config.d.ts.map +1 -1
- package/dist/wizard/copy/core.d.ts.map +1 -1
- package/dist/wizard/copy/core.js +4 -0
- package/dist/wizard/copy/core.js.map +1 -1
- package/dist/wizard/copy/index.d.ts.map +1 -1
- package/dist/wizard/copy/index.js +2 -0
- package/dist/wizard/copy/index.js.map +1 -1
- package/dist/wizard/copy/research.d.ts +3 -0
- package/dist/wizard/copy/research.d.ts.map +1 -0
- package/dist/wizard/copy/research.js +27 -0
- package/dist/wizard/copy/research.js.map +1 -0
- package/dist/wizard/copy/types.d.ts +5 -1
- package/dist/wizard/copy/types.d.ts.map +1 -1
- package/dist/wizard/flags.d.ts +7 -1
- package/dist/wizard/flags.d.ts.map +1 -1
- package/dist/wizard/questions.d.ts +4 -2
- package/dist/wizard/questions.d.ts.map +1 -1
- package/dist/wizard/questions.js +27 -1
- package/dist/wizard/questions.js.map +1 -1
- package/dist/wizard/questions.test.js +51 -0
- package/dist/wizard/questions.test.js.map +1 -1
- package/dist/wizard/wizard.d.ts +3 -2
- package/dist/wizard/wizard.d.ts.map +1 -1
- package/dist/wizard/wizard.js +3 -1
- package/dist/wizard/wizard.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: research-quant-requirements
|
|
3
|
+
description: Trading system research requirements including strategy hypothesis definition, market regime assumptions, risk budgets, data requirements, and performance targets
|
|
4
|
+
topics: [research, quant-finance, requirements, hypothesis, risk-budget, performance-targets, validation]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
Trading strategy research requires requirements that go far beyond "find a profitable strategy." Every research project must define the strategy hypothesis with falsifiable predictions, declare market regime assumptions that bound the strategy's expected operating environment, establish risk budgets that constrain position sizing and drawdown limits, specify data requirements including quality standards and survivorship-bias-free universes, set performance targets with statistical significance thresholds, and mandate out-of-sample validation protocols. Without these constraints, a backtest will always find something that looks good on historical data -- the question is whether it will survive contact with live markets.
|
|
8
|
+
|
|
9
|
+
## Summary
|
|
10
|
+
|
|
11
|
+
Define trading research requirements as structured hypotheses with quantitative success criteria (Sharpe > X, max drawdown < Y%), explicit market regime assumptions (trending, mean-reverting, crisis), risk budgets (max position size, sector exposure, portfolio heat), data requirements (instrument universe, frequency, lookback period, survivorship-bias-free), and mandatory out-of-sample validation windows. Separate primary optimization targets from hard guardrails. Require statistical significance testing (minimum trade count, bootstrap confidence intervals) before any strategy is considered validated.
|
|
12
|
+
|
|
13
|
+
## Deep Guidance
|
|
14
|
+
|
|
15
|
+
### Strategy Hypothesis Structure
|
|
16
|
+
|
|
17
|
+
Every quant research hypothesis must specify the market inefficiency being exploited, the mechanism by which the strategy captures it, and the conditions under which it should fail:
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
# configs/quant_hypothesis.py
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
from enum import Enum
|
|
23
|
+
|
|
24
|
+
class MarketRegime(Enum):
|
|
25
|
+
TRENDING = "trending"
|
|
26
|
+
MEAN_REVERTING = "mean_reverting"
|
|
27
|
+
HIGH_VOLATILITY = "high_volatility"
|
|
28
|
+
LOW_VOLATILITY = "low_volatility"
|
|
29
|
+
CRISIS = "crisis"
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class StrategyHypothesis:
|
|
33
|
+
"""Structured hypothesis for a trading strategy research project."""
|
|
34
|
+
hypothesis_id: str
|
|
35
|
+
name: str
|
|
36
|
+
inefficiency: str # What market inefficiency is being exploited
|
|
37
|
+
mechanism: str # How the strategy captures the inefficiency
|
|
38
|
+
expected_regimes: list[MarketRegime] # Regimes where strategy should work
|
|
39
|
+
failure_regimes: list[MarketRegime] # Regimes where strategy should fail
|
|
40
|
+
instruments: list[str] # Target instrument universe
|
|
41
|
+
timeframe: str # Primary timeframe (e.g., "1D", "1H", "5m")
|
|
42
|
+
min_sharpe: float = 1.5 # Minimum Sharpe ratio target
|
|
43
|
+
max_drawdown: float = 0.15 # Maximum acceptable drawdown (15%)
|
|
44
|
+
min_trades: int = 100 # Minimum trades for statistical significance
|
|
45
|
+
oos_ratio: float = 0.3 # Out-of-sample data ratio
|
|
46
|
+
|
|
47
|
+
def validate(self) -> list[str]:
|
|
48
|
+
"""Check hypothesis completeness."""
|
|
49
|
+
issues = []
|
|
50
|
+
if not self.inefficiency:
|
|
51
|
+
issues.append("Must specify the market inefficiency being exploited")
|
|
52
|
+
if not self.failure_regimes:
|
|
53
|
+
issues.append("Must specify regimes where strategy is expected to fail")
|
|
54
|
+
if self.min_trades < 30:
|
|
55
|
+
issues.append("Minimum 30 trades required for statistical significance")
|
|
56
|
+
if self.oos_ratio < 0.2:
|
|
57
|
+
issues.append("Out-of-sample ratio should be at least 20%")
|
|
58
|
+
return issues
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# Example hypothesis
|
|
62
|
+
momentum_hypothesis = StrategyHypothesis(
|
|
63
|
+
hypothesis_id="H-001",
|
|
64
|
+
name="Adaptive momentum crossover",
|
|
65
|
+
inefficiency="Price trends persist due to institutional herding and gradual information diffusion",
|
|
66
|
+
mechanism="Dual moving average crossover with volatility-adaptive lookback periods",
|
|
67
|
+
expected_regimes=[MarketRegime.TRENDING, MarketRegime.LOW_VOLATILITY],
|
|
68
|
+
failure_regimes=[MarketRegime.MEAN_REVERTING, MarketRegime.CRISIS],
|
|
69
|
+
instruments=["SPY", "QQQ", "IWM", "EFA", "EEM"],
|
|
70
|
+
timeframe="1D",
|
|
71
|
+
min_sharpe=1.5,
|
|
72
|
+
max_drawdown=0.15,
|
|
73
|
+
min_trades=200,
|
|
74
|
+
oos_ratio=0.3,
|
|
75
|
+
)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
### Market Regime Assumptions
|
|
79
|
+
|
|
80
|
+
Strategies do not operate in a vacuum. Document which market regimes the strategy targets and how regime detection will gate live deployment:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
# configs/regime_assumptions.py
|
|
84
|
+
from dataclasses import dataclass
|
|
85
|
+
|
|
86
|
+
@dataclass
|
|
87
|
+
class RegimeAssumption:
|
|
88
|
+
"""A documented assumption about market regime behavior."""
|
|
89
|
+
regime: str
|
|
90
|
+
volatility_range: tuple[float, float] # Annualized vol range
|
|
91
|
+
correlation_expectation: str # e.g., "cross-asset correlations < 0.5"
|
|
92
|
+
historical_frequency: float # Fraction of time market is in this regime
|
|
93
|
+
strategy_expectation: str # What we expect the strategy to do
|
|
94
|
+
|
|
95
|
+
regime_assumptions = [
|
|
96
|
+
RegimeAssumption(
|
|
97
|
+
regime="trending_low_vol",
|
|
98
|
+
volatility_range=(0.08, 0.18),
|
|
99
|
+
correlation_expectation="Sector correlations moderate (0.3-0.6)",
|
|
100
|
+
historical_frequency=0.45,
|
|
101
|
+
strategy_expectation="Primary alpha generation, Sharpe > 2.0",
|
|
102
|
+
),
|
|
103
|
+
RegimeAssumption(
|
|
104
|
+
regime="mean_reverting",
|
|
105
|
+
volatility_range=(0.10, 0.22),
|
|
106
|
+
correlation_expectation="Cross-asset correlations low (<0.3)",
|
|
107
|
+
historical_frequency=0.25,
|
|
108
|
+
strategy_expectation="Flat to slightly negative, drawdown < 5%",
|
|
109
|
+
),
|
|
110
|
+
RegimeAssumption(
|
|
111
|
+
regime="crisis",
|
|
112
|
+
volatility_range=(0.30, 0.80),
|
|
113
|
+
correlation_expectation="Correlations spike to > 0.8",
|
|
114
|
+
historical_frequency=0.10,
|
|
115
|
+
strategy_expectation="Strategy should be flat (kill switch active)",
|
|
116
|
+
),
|
|
117
|
+
]
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
### Risk Budget Requirements
|
|
121
|
+
|
|
122
|
+
Risk budgets define the hard constraints that no backtest optimization is allowed to violate. These are guardrails, not optimization targets:
|
|
123
|
+
|
|
124
|
+
| Risk Dimension | Typical Constraint | Enforcement |
|
|
125
|
+
|---------------|-------------------|-------------|
|
|
126
|
+
| Max position size | 5-10% of portfolio per instrument | Pre-trade check |
|
|
127
|
+
| Max sector exposure | 25-30% in any single sector | Pre-trade check |
|
|
128
|
+
| Max portfolio heat | 2-3% total risk per day | Daily limit |
|
|
129
|
+
| Max drawdown | 15-25% from peak | Kill switch |
|
|
130
|
+
| Max correlation to benchmark | 0.7 (if seeking alpha, not beta) | Monthly review |
|
|
131
|
+
| Min cash reserve | 10-20% uninvested | Rebalance trigger |
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
# configs/risk_budget.py
|
|
135
|
+
from dataclasses import dataclass
|
|
136
|
+
|
|
137
|
+
@dataclass
|
|
138
|
+
class RiskBudget:
|
|
139
|
+
"""Hard risk constraints for strategy research."""
|
|
140
|
+
max_position_pct: float = 0.10 # 10% max per instrument
|
|
141
|
+
max_sector_pct: float = 0.30 # 30% max per sector
|
|
142
|
+
max_daily_risk_pct: float = 0.02 # 2% portfolio heat per day
|
|
143
|
+
max_drawdown_pct: float = 0.20 # 20% max drawdown (kill switch)
|
|
144
|
+
max_leverage: float = 1.0 # No leverage by default
|
|
145
|
+
min_cash_pct: float = 0.10 # 10% cash reserve
|
|
146
|
+
max_correlation_to_benchmark: float = 0.70
|
|
147
|
+
|
|
148
|
+
def check_position(self, position_pct: float) -> bool:
|
|
149
|
+
return position_pct <= self.max_position_pct
|
|
150
|
+
|
|
151
|
+
def check_drawdown(self, current_drawdown: float) -> bool:
|
|
152
|
+
return current_drawdown <= self.max_drawdown_pct
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
### Data Requirements Specification
|
|
156
|
+
|
|
157
|
+
Data requirements must be explicit about instrument universe, frequency, lookback period, and quality standards:
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
# configs/data_requirements.py
|
|
161
|
+
from dataclasses import dataclass, field
|
|
162
|
+
from datetime import date
|
|
163
|
+
|
|
164
|
+
@dataclass
|
|
165
|
+
class DataRequirements:
|
|
166
|
+
"""Minimum data requirements for the research project."""
|
|
167
|
+
instruments: list[str] # Ticker symbols or identifiers
|
|
168
|
+
frequency: str # "1m", "5m", "1H", "1D"
|
|
169
|
+
start_date: date # Earliest required data point
|
|
170
|
+
end_date: date # Latest required data point
|
|
171
|
+
survivorship_bias_free: bool = True # Require delisted stocks
|
|
172
|
+
adjusted_for_splits: bool = True # Corporate action adjusted
|
|
173
|
+
adjusted_for_dividends: bool = True # Dividend adjusted
|
|
174
|
+
min_history_days: int = 252 * 5 # 5 years minimum
|
|
175
|
+
max_gap_days: int = 5 # Max consecutive missing days
|
|
176
|
+
required_fields: list[str] = field(
|
|
177
|
+
default_factory=lambda: ["open", "high", "low", "close", "volume"]
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
def validate_dataset(self, df) -> list[str]:
|
|
181
|
+
"""Validate a dataset meets requirements."""
|
|
182
|
+
issues = []
|
|
183
|
+
for col in self.required_fields:
|
|
184
|
+
if col not in df.columns:
|
|
185
|
+
issues.append(f"Missing required field: {col}")
|
|
186
|
+
if df.isnull().any().any():
|
|
187
|
+
null_pct = df.isnull().sum().sum() / df.size * 100
|
|
188
|
+
if null_pct > 1.0:
|
|
189
|
+
issues.append(f"Null values: {null_pct:.1f}% (max 1%)")
|
|
190
|
+
return issues
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
### Performance Target Framework
|
|
194
|
+
|
|
195
|
+
Performance targets must distinguish between the optimization target (one metric), secondary indicators (useful but not optimized), and hard guardrails (never violated):
|
|
196
|
+
|
|
197
|
+
| Category | Metric | Target | Rationale |
|
|
198
|
+
|----------|--------|--------|-----------|
|
|
199
|
+
| **Primary** | Sharpe ratio (OOS) | > 1.5 | Risk-adjusted return is the universal benchmark |
|
|
200
|
+
| Secondary | Sortino ratio | > 2.0 | Penalizes downside more than upside volatility |
|
|
201
|
+
| Secondary | Win rate | > 45% | Indicates strategy consistency |
|
|
202
|
+
| Secondary | Profit factor | > 1.5 | Gross profit / gross loss |
|
|
203
|
+
| **Guardrail** | Max drawdown | < 20% | Capital preservation |
|
|
204
|
+
| **Guardrail** | Min trades | > 100 | Statistical significance |
|
|
205
|
+
| **Guardrail** | Max consecutive losses | < 10 | Psychological tolerance |
|
|
206
|
+
|
|
207
|
+
### Out-of-Sample Validation Requirements
|
|
208
|
+
|
|
209
|
+
No strategy is considered validated until it passes out-of-sample testing. The validation protocol must be defined before the first experiment runs:
|
|
210
|
+
|
|
211
|
+
1. **Time-based split**: Reserve the most recent 30% of data as out-of-sample. Never touch it during development.
|
|
212
|
+
2. **Walk-forward validation**: Use rolling windows (e.g., 252-day train, 63-day test) to simulate realistic deployment.
|
|
213
|
+
3. **Multiple market regimes**: OOS period must include at least two distinct regime types.
|
|
214
|
+
4. **Statistical significance**: Bootstrap the strategy returns (1000+ iterations) and require the 5th percentile Sharpe > 0.
|
|
215
|
+
5. **Comparison to benchmarks**: Beat buy-and-hold AND a simple momentum benchmark on a risk-adjusted basis.
|
|
216
|
+
|
|
217
|
+
### Requirements Anti-Patterns in Quant Research
|
|
218
|
+
|
|
219
|
+
- **Overfitting the requirements**: Setting Sharpe target to match exactly what your best backtest achieved.
|
|
220
|
+
- **Survivorship bias in instrument selection**: Using today's S&P 500 constituents for a 20-year backtest.
|
|
221
|
+
- **Ignoring transaction costs**: A strategy that trades 50 times per day needs very different targets than one that trades monthly.
|
|
222
|
+
- **No regime awareness**: "Works in all market conditions" is a red flag -- no strategy does.
|
|
223
|
+
- **Optimizing guardrails**: If max drawdown is a guardrail, do not run optimizations that minimize drawdown -- it becomes a de facto primary target and distorts the search.
|
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: research-quant-risk
|
|
3
|
+
description: Risk management for trading research including regime detection, tail risk measures, correlation breakdown, position limits, drawdown controls, and kill switches
|
|
4
|
+
topics: [research, quant-finance, risk, regime-detection, tail-risk, var, cvar, correlation, drawdown-controls, kill-switch]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
Risk management in quantitative finance is not an afterthought bolted onto a profitable strategy -- it is the primary constraint that determines whether a strategy survives long enough to realize its edge. Markets exhibit non-stationary behavior: correlations spike during crises, volatility clusters in ways that violate normal distribution assumptions, and tail events occur far more frequently than Gaussian models predict. A strategy without regime awareness, tail risk measurement, position limits, and automated kill switches is not a strategy -- it is a gamble.
|
|
8
|
+
|
|
9
|
+
## Summary
|
|
10
|
+
|
|
11
|
+
Implement risk management as a layered system: regime detection (HMM, volatility clustering) determines the current market state and gates strategy behavior; tail risk measures (VaR, CVaR, stress testing) quantify worst-case scenarios beyond what standard deviation captures; position limits (max position size, sector exposure, concentration) prevent over-commitment; drawdown controls (max drawdown threshold, recovery rules) cap cumulative losses; and kill switches (automated circuit breakers) halt all trading when conditions exceed safe operating parameters. Every layer operates independently -- if one fails, the others still protect capital.
|
|
12
|
+
|
|
13
|
+
## Deep Guidance
|
|
14
|
+
|
|
15
|
+
### Regime Detection
|
|
16
|
+
|
|
17
|
+
Market regimes (trending, mean-reverting, high-volatility, crisis) fundamentally change strategy performance. Detect regimes to gate strategy behavior:
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
# risk/regime_detection.py
|
|
21
|
+
import numpy as np
|
|
22
|
+
import pandas as pd
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
from enum import Enum
|
|
25
|
+
|
|
26
|
+
class Regime(Enum):
|
|
27
|
+
LOW_VOL = "low_volatility"
|
|
28
|
+
NORMAL = "normal"
|
|
29
|
+
HIGH_VOL = "high_volatility"
|
|
30
|
+
CRISIS = "crisis"
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class RegimeState:
|
|
34
|
+
"""Current regime classification with confidence."""
|
|
35
|
+
regime: Regime
|
|
36
|
+
confidence: float # 0-1
|
|
37
|
+
volatility: float # Current annualized volatility
|
|
38
|
+
vol_percentile: float # Percentile vs. history
|
|
39
|
+
|
|
40
|
+
class VolatilityRegimeDetector:
|
|
41
|
+
"""
|
|
42
|
+
Simple volatility-based regime detector.
|
|
43
|
+
|
|
44
|
+
Uses realized volatility percentiles relative to history.
|
|
45
|
+
More robust than HMM for live deployment (no fitting required).
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(
|
|
49
|
+
self,
|
|
50
|
+
lookback: int = 20,
|
|
51
|
+
history_window: int = 252 * 5,
|
|
52
|
+
crisis_percentile: float = 95,
|
|
53
|
+
high_vol_percentile: float = 75,
|
|
54
|
+
low_vol_percentile: float = 25,
|
|
55
|
+
):
|
|
56
|
+
self.lookback = lookback
|
|
57
|
+
self.history_window = history_window
|
|
58
|
+
self.crisis_pct = crisis_percentile
|
|
59
|
+
self.high_vol_pct = high_vol_percentile
|
|
60
|
+
self.low_vol_pct = low_vol_percentile
|
|
61
|
+
|
|
62
|
+
def detect(self, returns: pd.Series) -> RegimeState:
|
|
63
|
+
"""Classify current regime from return history."""
|
|
64
|
+
recent_vol = returns.tail(self.lookback).std() * np.sqrt(252)
|
|
65
|
+
historical_vols = (
|
|
66
|
+
returns.rolling(self.lookback).std() * np.sqrt(252)
|
|
67
|
+
).dropna().tail(self.history_window)
|
|
68
|
+
|
|
69
|
+
percentile = (historical_vols < recent_vol).mean() * 100
|
|
70
|
+
|
|
71
|
+
if percentile >= self.crisis_pct:
|
|
72
|
+
regime = Regime.CRISIS
|
|
73
|
+
confidence = min((percentile - self.crisis_pct) / 5 + 0.8, 1.0)
|
|
74
|
+
elif percentile >= self.high_vol_pct:
|
|
75
|
+
regime = Regime.HIGH_VOL
|
|
76
|
+
confidence = 0.7
|
|
77
|
+
elif percentile <= self.low_vol_pct:
|
|
78
|
+
regime = Regime.LOW_VOL
|
|
79
|
+
confidence = 0.7
|
|
80
|
+
else:
|
|
81
|
+
regime = Regime.NORMAL
|
|
82
|
+
confidence = 0.8
|
|
83
|
+
|
|
84
|
+
return RegimeState(
|
|
85
|
+
regime=regime,
|
|
86
|
+
confidence=confidence,
|
|
87
|
+
volatility=recent_vol,
|
|
88
|
+
vol_percentile=percentile,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class HMMRegimeDetector:
|
|
93
|
+
"""
|
|
94
|
+
Hidden Markov Model regime detector.
|
|
95
|
+
|
|
96
|
+
Fits a 2-3 state HMM to return series. Better for research
|
|
97
|
+
(offline analysis) than live trading (requires refitting).
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
def __init__(self, n_states: int = 3, lookback: int = 252 * 3):
|
|
101
|
+
self.n_states = n_states
|
|
102
|
+
self.lookback = lookback
|
|
103
|
+
self.model = None
|
|
104
|
+
|
|
105
|
+
def fit(self, returns: pd.Series) -> None:
|
|
106
|
+
"""Fit HMM to historical returns."""
|
|
107
|
+
try:
|
|
108
|
+
from hmmlearn.hmm import GaussianHMM
|
|
109
|
+
except ImportError:
|
|
110
|
+
raise ImportError("pip install hmmlearn for HMM regime detection")
|
|
111
|
+
|
|
112
|
+
data = returns.tail(self.lookback).values.reshape(-1, 1)
|
|
113
|
+
self.model = GaussianHMM(
|
|
114
|
+
n_components=self.n_states,
|
|
115
|
+
covariance_type="full",
|
|
116
|
+
n_iter=100,
|
|
117
|
+
random_state=42,
|
|
118
|
+
)
|
|
119
|
+
self.model.fit(data)
|
|
120
|
+
|
|
121
|
+
def predict(self, returns: pd.Series) -> np.ndarray:
|
|
122
|
+
"""Predict regime states for each time step."""
|
|
123
|
+
if self.model is None:
|
|
124
|
+
raise RuntimeError("Call fit() before predict()")
|
|
125
|
+
data = returns.values.reshape(-1, 1)
|
|
126
|
+
return self.model.predict(data)
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### Tail Risk Measures
|
|
130
|
+
|
|
131
|
+
Standard deviation understates risk because market returns have fat tails. Use VaR and CVaR for tail risk:
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
# risk/tail_risk.py
|
|
135
|
+
import numpy as np
|
|
136
|
+
import pandas as pd
|
|
137
|
+
|
|
138
|
+
def value_at_risk(
|
|
139
|
+
returns: pd.Series,
|
|
140
|
+
confidence: float = 0.95,
|
|
141
|
+
method: str = "historical",
|
|
142
|
+
lookback: int = 252,
|
|
143
|
+
) -> float:
|
|
144
|
+
"""
|
|
145
|
+
Value at Risk — maximum expected loss at a given confidence level.
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
confidence: Confidence level (e.g., 0.95 = 95%).
|
|
149
|
+
method: "historical" (percentile) or "parametric" (Gaussian).
|
|
150
|
+
lookback: Number of observations to use.
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
VaR as a negative number (loss).
|
|
154
|
+
"""
|
|
155
|
+
data = returns.tail(lookback).dropna()
|
|
156
|
+
|
|
157
|
+
if method == "historical":
|
|
158
|
+
return float(np.percentile(data, (1 - confidence) * 100))
|
|
159
|
+
elif method == "parametric":
|
|
160
|
+
from scipy.stats import norm
|
|
161
|
+
z = norm.ppf(1 - confidence)
|
|
162
|
+
return float(data.mean() + z * data.std())
|
|
163
|
+
else:
|
|
164
|
+
raise ValueError(f"Unknown VaR method: {method}")
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def conditional_var(
|
|
168
|
+
returns: pd.Series,
|
|
169
|
+
confidence: float = 0.95,
|
|
170
|
+
lookback: int = 252,
|
|
171
|
+
) -> float:
|
|
172
|
+
"""
|
|
173
|
+
Conditional VaR (Expected Shortfall) — average loss beyond VaR.
|
|
174
|
+
|
|
175
|
+
CVaR answers: "When we do lose more than VaR, how bad is it on average?"
|
|
176
|
+
Always worse than VaR, captures tail severity.
|
|
177
|
+
"""
|
|
178
|
+
data = returns.tail(lookback).dropna()
|
|
179
|
+
var = value_at_risk(data, confidence, method="historical", lookback=lookback)
|
|
180
|
+
tail_losses = data[data <= var]
|
|
181
|
+
return float(tail_losses.mean()) if len(tail_losses) > 0 else var
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def stress_test(
|
|
185
|
+
portfolio_returns: pd.Series,
|
|
186
|
+
scenarios: dict[str, tuple[str, str]],
|
|
187
|
+
market_data: pd.DataFrame,
|
|
188
|
+
) -> dict[str, dict]:
|
|
189
|
+
"""
|
|
190
|
+
Stress test portfolio against historical crisis scenarios.
|
|
191
|
+
|
|
192
|
+
Args:
|
|
193
|
+
scenarios: Dict of scenario_name -> (start_date, end_date).
|
|
194
|
+
"""
|
|
195
|
+
results = {}
|
|
196
|
+
for name, (start, end) in scenarios.items():
|
|
197
|
+
mask = (portfolio_returns.index >= start) & (portfolio_returns.index <= end)
|
|
198
|
+
scenario_returns = portfolio_returns[mask]
|
|
199
|
+
|
|
200
|
+
if len(scenario_returns) == 0:
|
|
201
|
+
continue
|
|
202
|
+
|
|
203
|
+
results[name] = {
|
|
204
|
+
"total_return": float((1 + scenario_returns).prod() - 1),
|
|
205
|
+
"max_drawdown": float(_max_dd(scenario_returns)),
|
|
206
|
+
"worst_day": float(scenario_returns.min()),
|
|
207
|
+
"volatility": float(scenario_returns.std() * np.sqrt(252)),
|
|
208
|
+
"days": len(scenario_returns),
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
return results
|
|
212
|
+
|
|
213
|
+
def _max_dd(returns: pd.Series) -> float:
|
|
214
|
+
equity = (1 + returns).cumprod()
|
|
215
|
+
return float((equity / equity.cummax() - 1).min())
|
|
216
|
+
|
|
217
|
+
# Standard stress test scenarios
|
|
218
|
+
CRISIS_SCENARIOS = {
|
|
219
|
+
"gfc_2008": ("2008-09-01", "2009-03-31"),
|
|
220
|
+
"covid_2020": ("2020-02-19", "2020-03-23"),
|
|
221
|
+
"dot_com_2000": ("2000-03-10", "2002-10-09"),
|
|
222
|
+
"flash_crash_2010": ("2010-05-06", "2010-05-06"),
|
|
223
|
+
"vol_shock_2018": ("2018-02-01", "2018-02-09"),
|
|
224
|
+
"rate_hike_2022": ("2022-01-03", "2022-10-12"),
|
|
225
|
+
}
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
### Correlation Breakdown
|
|
229
|
+
|
|
230
|
+
During crises, asset correlations spike toward 1.0, destroying diversification benefits exactly when they are most needed:
|
|
231
|
+
|
|
232
|
+
```python
|
|
233
|
+
# risk/correlation.py
|
|
234
|
+
import numpy as np
|
|
235
|
+
import pandas as pd
|
|
236
|
+
|
|
237
|
+
def rolling_correlation_matrix(
|
|
238
|
+
returns: pd.DataFrame,
|
|
239
|
+
window: int = 60,
|
|
240
|
+
) -> pd.DataFrame:
|
|
241
|
+
"""Compute rolling correlation matrix (returns latest window)."""
|
|
242
|
+
return returns.tail(window).corr()
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def detect_correlation_spike(
|
|
246
|
+
returns: pd.DataFrame,
|
|
247
|
+
window: int = 20,
|
|
248
|
+
history_window: int = 252,
|
|
249
|
+
threshold_percentile: float = 90,
|
|
250
|
+
) -> dict:
|
|
251
|
+
"""
|
|
252
|
+
Detect when average pairwise correlation exceeds historical norms.
|
|
253
|
+
|
|
254
|
+
Returns:
|
|
255
|
+
Dict with current_avg_corr, historical_percentile, is_spike.
|
|
256
|
+
"""
|
|
257
|
+
# Current average pairwise correlation
|
|
258
|
+
current_corr = returns.tail(window).corr()
|
|
259
|
+
mask = np.triu(np.ones_like(current_corr, dtype=bool), k=1)
|
|
260
|
+
current_avg = float(current_corr.values[mask].mean())
|
|
261
|
+
|
|
262
|
+
# Historical rolling average correlations
|
|
263
|
+
rolling_avgs = []
|
|
264
|
+
for i in range(history_window, len(returns)):
|
|
265
|
+
chunk = returns.iloc[i - window:i]
|
|
266
|
+
corr = chunk.corr()
|
|
267
|
+
avg = float(corr.values[mask].mean())
|
|
268
|
+
rolling_avgs.append(avg)
|
|
269
|
+
|
|
270
|
+
percentile = (np.array(rolling_avgs) < current_avg).mean() * 100
|
|
271
|
+
|
|
272
|
+
return {
|
|
273
|
+
"current_avg_corr": current_avg,
|
|
274
|
+
"historical_percentile": percentile,
|
|
275
|
+
"is_spike": percentile > threshold_percentile,
|
|
276
|
+
}
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
### Position Limits
|
|
280
|
+
|
|
281
|
+
Hard limits that prevent any single position or concentration from threatening the portfolio:
|
|
282
|
+
|
|
283
|
+
```python
|
|
284
|
+
# risk/position_limits.py
|
|
285
|
+
from dataclasses import dataclass
|
|
286
|
+
|
|
287
|
+
@dataclass
|
|
288
|
+
class PositionLimits:
|
|
289
|
+
"""Hard position limits enforced before every trade."""
|
|
290
|
+
max_position_pct: float = 0.10 # 10% max per instrument
|
|
291
|
+
max_sector_pct: float = 0.30 # 30% max per sector
|
|
292
|
+
max_concentration_pct: float = 0.50 # 50% max in top 3 positions
|
|
293
|
+
max_gross_exposure: float = 1.0 # 100% (no leverage)
|
|
294
|
+
max_net_exposure: float = 0.50 # 50% max net long or short
|
|
295
|
+
max_single_day_turnover: float = 0.30 # 30% max daily turnover
|
|
296
|
+
|
|
297
|
+
def check_trade(
|
|
298
|
+
self,
|
|
299
|
+
proposed_position_pct: float,
|
|
300
|
+
current_sector_pct: float,
|
|
301
|
+
current_gross_exposure: float,
|
|
302
|
+
) -> tuple[bool, str]:
|
|
303
|
+
"""Check if a proposed trade violates any limit."""
|
|
304
|
+
if proposed_position_pct > self.max_position_pct:
|
|
305
|
+
return False, f"Position {proposed_position_pct:.1%} exceeds max {self.max_position_pct:.1%}"
|
|
306
|
+
if current_sector_pct > self.max_sector_pct:
|
|
307
|
+
return False, f"Sector exposure {current_sector_pct:.1%} exceeds max {self.max_sector_pct:.1%}"
|
|
308
|
+
if current_gross_exposure > self.max_gross_exposure:
|
|
309
|
+
return False, f"Gross exposure {current_gross_exposure:.1%} exceeds max {self.max_gross_exposure:.1%}"
|
|
310
|
+
return True, "OK"
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
### Drawdown Controls
|
|
314
|
+
|
|
315
|
+
Drawdown controls cap cumulative losses and enforce recovery discipline:
|
|
316
|
+
|
|
317
|
+
```python
|
|
318
|
+
# risk/drawdown_controls.py
|
|
319
|
+
from dataclasses import dataclass
|
|
320
|
+
from enum import Enum
|
|
321
|
+
|
|
322
|
+
class DrawdownAction(Enum):
|
|
323
|
+
NORMAL = "normal" # Full position sizing
|
|
324
|
+
REDUCED = "reduced" # Half position sizes
|
|
325
|
+
HALTED = "halted" # No new positions (close existing)
|
|
326
|
+
|
|
327
|
+
@dataclass
|
|
328
|
+
class DrawdownController:
|
|
329
|
+
"""Multi-tier drawdown control system."""
|
|
330
|
+
warn_threshold: float = 0.10 # 10% — log warning, reduce size
|
|
331
|
+
halt_threshold: float = 0.15 # 15% — stop opening new positions
|
|
332
|
+
kill_threshold: float = 0.20 # 20% — close everything
|
|
333
|
+
recovery_required: float = 0.50 # Must recover 50% of drawdown to resume
|
|
334
|
+
|
|
335
|
+
def evaluate(self, current_drawdown: float,
|
|
336
|
+
recovered_pct: float = 0.0) -> DrawdownAction:
|
|
337
|
+
"""
|
|
338
|
+
Determine trading action based on current drawdown.
|
|
339
|
+
|
|
340
|
+
Args:
|
|
341
|
+
current_drawdown: Current drawdown as positive fraction (e.g., 0.12 = 12%).
|
|
342
|
+
recovered_pct: If in recovery mode, fraction of drawdown recovered.
|
|
343
|
+
"""
|
|
344
|
+
dd = abs(current_drawdown)
|
|
345
|
+
|
|
346
|
+
if dd >= self.kill_threshold:
|
|
347
|
+
return DrawdownAction.HALTED
|
|
348
|
+
|
|
349
|
+
if dd >= self.halt_threshold:
|
|
350
|
+
if recovered_pct < self.recovery_required:
|
|
351
|
+
return DrawdownAction.HALTED
|
|
352
|
+
return DrawdownAction.REDUCED
|
|
353
|
+
|
|
354
|
+
if dd >= self.warn_threshold:
|
|
355
|
+
return DrawdownAction.REDUCED
|
|
356
|
+
|
|
357
|
+
return DrawdownAction.NORMAL
|
|
358
|
+
|
|
359
|
+
def scale_factor(self, action: DrawdownAction) -> float:
|
|
360
|
+
"""Return position size multiplier for the given action."""
|
|
361
|
+
return {
|
|
362
|
+
DrawdownAction.NORMAL: 1.0,
|
|
363
|
+
DrawdownAction.REDUCED: 0.5,
|
|
364
|
+
DrawdownAction.HALTED: 0.0,
|
|
365
|
+
}[action]
|
|
366
|
+
```
|
|
367
|
+
|
|
368
|
+
### Kill Switches
|
|
369
|
+
|
|
370
|
+
Kill switches are automated circuit breakers that halt all activity when conditions become dangerous. They are the last line of defense:
|
|
371
|
+
|
|
372
|
+
```python
|
|
373
|
+
# risk/kill_switch.py
|
|
374
|
+
import logging
|
|
375
|
+
from dataclasses import dataclass
|
|
376
|
+
from datetime import datetime, timedelta
|
|
377
|
+
|
|
378
|
+
logger = logging.getLogger(__name__)
|
|
379
|
+
|
|
380
|
+
@dataclass
|
|
381
|
+
class KillSwitchConfig:
|
|
382
|
+
"""Automated circuit breaker configuration."""
|
|
383
|
+
max_daily_loss_pct: float = 0.05 # 5% daily loss limit
|
|
384
|
+
max_drawdown_pct: float = 0.20 # 20% total drawdown limit
|
|
385
|
+
max_consecutive_losses: int = 10 # Stop after 10 straight losses
|
|
386
|
+
max_volatility_multiple: float = 3.0 # 3x normal vol = halt
|
|
387
|
+
cooldown_period: timedelta = timedelta(hours=24) # Wait before resuming
|
|
388
|
+
|
|
389
|
+
class KillSwitch:
|
|
390
|
+
"""
|
|
391
|
+
Automated trading halt system.
|
|
392
|
+
|
|
393
|
+
Evaluates multiple independent triggers. If ANY trigger fires,
|
|
394
|
+
all trading stops immediately. Requires manual acknowledgment
|
|
395
|
+
or cooldown period before resuming.
|
|
396
|
+
"""
|
|
397
|
+
|
|
398
|
+
def __init__(self, config: KillSwitchConfig):
|
|
399
|
+
self.config = config
|
|
400
|
+
self.is_active = False
|
|
401
|
+
self.trigger_reason = ""
|
|
402
|
+
self.triggered_at: datetime | None = None
|
|
403
|
+
|
|
404
|
+
def check(
|
|
405
|
+
self,
|
|
406
|
+
daily_pnl_pct: float,
|
|
407
|
+
total_drawdown_pct: float,
|
|
408
|
+
consecutive_losses: int,
|
|
409
|
+
current_vol_multiple: float,
|
|
410
|
+
) -> bool:
|
|
411
|
+
"""
|
|
412
|
+
Check all kill switch triggers.
|
|
413
|
+
|
|
414
|
+
Returns True if trading should be halted.
|
|
415
|
+
"""
|
|
416
|
+
if abs(daily_pnl_pct) >= self.config.max_daily_loss_pct:
|
|
417
|
+
self._trigger(f"Daily loss {daily_pnl_pct:.1%} exceeds {self.config.max_daily_loss_pct:.1%}")
|
|
418
|
+
return True
|
|
419
|
+
|
|
420
|
+
if abs(total_drawdown_pct) >= self.config.max_drawdown_pct:
|
|
421
|
+
self._trigger(f"Drawdown {total_drawdown_pct:.1%} exceeds {self.config.max_drawdown_pct:.1%}")
|
|
422
|
+
return True
|
|
423
|
+
|
|
424
|
+
if consecutive_losses >= self.config.max_consecutive_losses:
|
|
425
|
+
self._trigger(f"{consecutive_losses} consecutive losses")
|
|
426
|
+
return True
|
|
427
|
+
|
|
428
|
+
if current_vol_multiple >= self.config.max_volatility_multiple:
|
|
429
|
+
self._trigger(f"Volatility {current_vol_multiple:.1f}x normal")
|
|
430
|
+
return True
|
|
431
|
+
|
|
432
|
+
return False
|
|
433
|
+
|
|
434
|
+
def _trigger(self, reason: str) -> None:
|
|
435
|
+
self.is_active = True
|
|
436
|
+
self.trigger_reason = reason
|
|
437
|
+
self.triggered_at = datetime.now()
|
|
438
|
+
logger.critical("KILL SWITCH ACTIVATED: %s", reason)
|
|
439
|
+
|
|
440
|
+
def can_resume(self) -> bool:
|
|
441
|
+
"""Check if cooldown period has elapsed."""
|
|
442
|
+
if not self.is_active or self.triggered_at is None:
|
|
443
|
+
return True
|
|
444
|
+
elapsed = datetime.now() - self.triggered_at
|
|
445
|
+
return elapsed >= self.config.cooldown_period
|
|
446
|
+
|
|
447
|
+
def reset(self, acknowledge: bool = False) -> None:
|
|
448
|
+
"""Reset kill switch (requires explicit acknowledgment)."""
|
|
449
|
+
if not acknowledge:
|
|
450
|
+
raise ValueError("Kill switch reset requires explicit acknowledgment")
|
|
451
|
+
logger.warning("Kill switch reset by operator (was: %s)", self.trigger_reason)
|
|
452
|
+
self.is_active = False
|
|
453
|
+
self.trigger_reason = ""
|
|
454
|
+
self.triggered_at = None
|
|
455
|
+
```
|
|
456
|
+
|
|
457
|
+
### Risk Management Layering
|
|
458
|
+
|
|
459
|
+
Risk controls operate in layers, each independent of the others:
|
|
460
|
+
|
|
461
|
+
| Layer | Check | Frequency | Response |
|
|
462
|
+
|-------|-------|-----------|----------|
|
|
463
|
+
| 1. Regime | Market state classification | Daily | Adjust strategy parameters |
|
|
464
|
+
| 2. Position limits | Per-trade constraint check | Pre-trade | Reject or resize trade |
|
|
465
|
+
| 3. Drawdown control | Cumulative loss monitoring | Per-trade | Reduce size or halt |
|
|
466
|
+
| 4. Kill switch | Emergency circuit breaker | Real-time | Halt all trading |
|
|
467
|
+
| 5. Stress test | Scenario-based exposure | Weekly | Adjust portfolio if stressed |
|
|
468
|
+
|
|
469
|
+
Never rely on a single layer. Assume each layer will fail eventually -- the redundancy is what protects capital.
|