sqa 0.0.24 → 0.0.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. checksums.yaml +4 -4
  2. data/.goose/memory/development.txt +3 -0
  3. data/.semver +6 -0
  4. data/ARCHITECTURE.md +648 -0
  5. data/CHANGELOG.md +95 -0
  6. data/CLAUDE.md +674 -0
  7. data/COMMITS.md +196 -0
  8. data/DATAFRAME_ARCHITECTURE_REVIEW.md +421 -0
  9. data/NEXT-STEPS.md +154 -0
  10. data/README.md +839 -265
  11. data/TASKS.md +358 -0
  12. data/TEST_RESULTS.md +140 -0
  13. data/TODO.md +42 -0
  14. data/_notes.txt +25 -0
  15. data/bin/sqa-console +11 -0
  16. data/data/talk_talk.json +103284 -0
  17. data/develop_summary.md +313 -0
  18. data/docs/advanced/backtesting.md +206 -0
  19. data/docs/advanced/ensemble.md +68 -0
  20. data/docs/advanced/fpop.md +153 -0
  21. data/docs/advanced/index.md +112 -0
  22. data/docs/advanced/multi-timeframe.md +67 -0
  23. data/docs/advanced/pattern-matcher.md +75 -0
  24. data/docs/advanced/portfolio-optimizer.md +79 -0
  25. data/docs/advanced/portfolio.md +166 -0
  26. data/docs/advanced/risk-management.md +210 -0
  27. data/docs/advanced/strategy-generator.md +158 -0
  28. data/docs/advanced/streaming.md +209 -0
  29. data/docs/ai_and_ml.md +80 -0
  30. data/docs/api/dataframe.md +1114 -0
  31. data/docs/api/index.md +126 -0
  32. data/docs/assets/css/custom.css +88 -0
  33. data/docs/assets/images/sqa.jpg +0 -0
  34. data/docs/assets/js/mathjax.js +18 -0
  35. data/docs/concepts/index.md +60 -0
  36. data/docs/contributing/index.md +60 -0
  37. data/docs/data-sources/index.md +66 -0
  38. data/docs/data_frame.md +316 -97
  39. data/docs/factors_that_impact_price.md +26 -0
  40. data/docs/finviz.md +11 -0
  41. data/docs/fx_pro_bit.md +25 -0
  42. data/docs/genetic_programming.md +104 -0
  43. data/docs/getting-started/index.md +107 -0
  44. data/docs/getting-started/installation.md +229 -0
  45. data/docs/getting-started/quick-start.md +244 -0
  46. data/docs/i_gotta_an_idea.md +22 -0
  47. data/docs/index.md +161 -0
  48. data/docs/indicators/index.md +97 -0
  49. data/docs/indicators.md +110 -24
  50. data/docs/options.md +8 -0
  51. data/docs/strategies/bollinger-bands.md +146 -0
  52. data/docs/strategies/consensus.md +64 -0
  53. data/docs/strategies/custom.md +310 -0
  54. data/docs/strategies/ema.md +53 -0
  55. data/docs/strategies/index.md +92 -0
  56. data/docs/strategies/kbs.md +164 -0
  57. data/docs/strategies/macd.md +96 -0
  58. data/docs/strategies/market-profile.md +54 -0
  59. data/docs/strategies/mean-reversion.md +58 -0
  60. data/docs/strategies/rsi.md +95 -0
  61. data/docs/strategies/sma.md +55 -0
  62. data/docs/strategies/stochastic.md +63 -0
  63. data/docs/strategies/volume-breakout.md +54 -0
  64. data/docs/tags.md +7 -0
  65. data/examples/README.md +354 -0
  66. data/examples/advanced_features_example.rb +350 -0
  67. data/examples/fpop_analysis_example.rb +191 -0
  68. data/examples/genetic_programming_example.rb +148 -0
  69. data/examples/kbs_strategy_example.rb +208 -0
  70. data/examples/pattern_context_example.rb +300 -0
  71. data/examples/rails_app/Gemfile +34 -0
  72. data/examples/rails_app/README.md +416 -0
  73. data/examples/rails_app/app/assets/javascripts/application.js +107 -0
  74. data/examples/rails_app/app/assets/stylesheets/application.css +659 -0
  75. data/examples/rails_app/app/controllers/analysis_controller.rb +11 -0
  76. data/examples/rails_app/app/controllers/api/v1/stocks_controller.rb +227 -0
  77. data/examples/rails_app/app/controllers/application_controller.rb +22 -0
  78. data/examples/rails_app/app/controllers/backtest_controller.rb +11 -0
  79. data/examples/rails_app/app/controllers/dashboard_controller.rb +21 -0
  80. data/examples/rails_app/app/controllers/portfolio_controller.rb +7 -0
  81. data/examples/rails_app/app/views/analysis/show.html.erb +209 -0
  82. data/examples/rails_app/app/views/backtest/show.html.erb +171 -0
  83. data/examples/rails_app/app/views/dashboard/index.html.erb +118 -0
  84. data/examples/rails_app/app/views/dashboard/show.html.erb +408 -0
  85. data/examples/rails_app/app/views/errors/show.html.erb +17 -0
  86. data/examples/rails_app/app/views/layouts/application.html.erb +60 -0
  87. data/examples/rails_app/app/views/portfolio/index.html.erb +33 -0
  88. data/examples/rails_app/bin/rails +6 -0
  89. data/examples/rails_app/config/application.rb +45 -0
  90. data/examples/rails_app/config/boot.rb +5 -0
  91. data/examples/rails_app/config/database.yml +18 -0
  92. data/examples/rails_app/config/environment.rb +11 -0
  93. data/examples/rails_app/config/routes.rb +26 -0
  94. data/examples/rails_app/config.ru +8 -0
  95. data/examples/realtime_stream_example.rb +274 -0
  96. data/examples/sinatra_app/Gemfile +42 -0
  97. data/examples/sinatra_app/Gemfile.lock +268 -0
  98. data/examples/sinatra_app/QUICKSTART.md +169 -0
  99. data/examples/sinatra_app/README.md +471 -0
  100. data/examples/sinatra_app/RUNNING_WITHOUT_TALIB.md +90 -0
  101. data/examples/sinatra_app/TROUBLESHOOTING.md +95 -0
  102. data/examples/sinatra_app/app.rb +404 -0
  103. data/examples/sinatra_app/config.ru +5 -0
  104. data/examples/sinatra_app/public/css/style.css +723 -0
  105. data/examples/sinatra_app/public/debug_macd.html +82 -0
  106. data/examples/sinatra_app/public/js/app.js +107 -0
  107. data/examples/sinatra_app/start.sh +53 -0
  108. data/examples/sinatra_app/views/analyze.erb +306 -0
  109. data/examples/sinatra_app/views/backtest.erb +325 -0
  110. data/examples/sinatra_app/views/dashboard.erb +831 -0
  111. data/examples/sinatra_app/views/error.erb +58 -0
  112. data/examples/sinatra_app/views/index.erb +118 -0
  113. data/examples/sinatra_app/views/layout.erb +61 -0
  114. data/examples/sinatra_app/views/portfolio.erb +43 -0
  115. data/examples/strategy_generator_example.rb +346 -0
  116. data/hsa_portfolio.csv +11 -0
  117. data/justfile +0 -0
  118. data/lib/api/alpha_vantage_api.rb +462 -0
  119. data/lib/sqa/backtest.rb +329 -0
  120. data/lib/sqa/data_frame/alpha_vantage.rb +51 -63
  121. data/lib/sqa/data_frame/data.rb +92 -0
  122. data/lib/sqa/data_frame/yahoo_finance.rb +35 -43
  123. data/lib/sqa/data_frame.rb +154 -243
  124. data/lib/sqa/ensemble.rb +359 -0
  125. data/lib/sqa/fpop.rb +199 -0
  126. data/lib/sqa/gp.rb +259 -0
  127. data/lib/sqa/indicator.rb +16 -6
  128. data/lib/sqa/init.rb +15 -8
  129. data/lib/sqa/market_regime.rb +240 -0
  130. data/lib/sqa/multi_timeframe.rb +379 -0
  131. data/lib/sqa/pattern_matcher.rb +497 -0
  132. data/lib/sqa/portfolio.rb +260 -6
  133. data/lib/sqa/portfolio_optimizer.rb +377 -0
  134. data/lib/sqa/risk_manager.rb +442 -0
  135. data/lib/sqa/seasonal_analyzer.rb +209 -0
  136. data/lib/sqa/sector_analyzer.rb +300 -0
  137. data/lib/sqa/stock.rb +131 -127
  138. data/lib/sqa/strategy/bollinger_bands.rb +42 -0
  139. data/lib/sqa/strategy/consensus.rb +5 -2
  140. data/lib/sqa/strategy/kbs_strategy.rb +470 -0
  141. data/lib/sqa/strategy/macd.rb +46 -0
  142. data/lib/sqa/strategy/mp.rb +1 -1
  143. data/lib/sqa/strategy/stochastic.rb +60 -0
  144. data/lib/sqa/strategy/volume_breakout.rb +57 -0
  145. data/lib/sqa/strategy.rb +5 -0
  146. data/lib/sqa/strategy_generator.rb +947 -0
  147. data/lib/sqa/stream.rb +361 -0
  148. data/lib/sqa/ticker.rb +9 -2
  149. data/lib/sqa/version.rb +1 -7
  150. data/lib/sqa.rb +35 -20
  151. data/main.just +81 -0
  152. data/mkdocs.yml +252 -0
  153. data/trace.log +0 -0
  154. metadata +265 -69
  155. data/bin/sqa +0 -6
  156. data/docs/alpha_vantage_technical_indicators.md +0 -62
  157. data/docs/average_true_range.md +0 -9
  158. data/docs/bollinger_bands.md +0 -15
  159. data/docs/candlestick_pattern_recognizer.md +0 -4
  160. data/docs/donchian_channel.md +0 -5
  161. data/docs/double_top_bottom_pattern.md +0 -3
  162. data/docs/exponential_moving_average.md +0 -19
  163. data/docs/fibonacci_retracement.md +0 -30
  164. data/docs/head_and_shoulders_pattern.md +0 -3
  165. data/docs/market_profile.md +0 -4
  166. data/docs/momentum.md +0 -19
  167. data/docs/moving_average_convergence_divergence.md +0 -23
  168. data/docs/peaks_and_valleys.md +0 -11
  169. data/docs/relative_strength_index.md +0 -6
  170. data/docs/simple_moving_average.md +0 -8
  171. data/docs/stochastic_oscillator.md +0 -4
  172. data/docs/ta_lib.md +0 -160
  173. data/docs/true_range.md +0 -12
  174. data/lib/patches/dry-cli.rb +0 -228
  175. data/lib/sqa/activity.rb +0 -10
  176. data/lib/sqa/cli.rb +0 -62
  177. data/lib/sqa/commands/analysis.rb +0 -309
  178. data/lib/sqa/commands/base.rb +0 -139
  179. data/lib/sqa/commands/web.rb +0 -199
  180. data/lib/sqa/commands.rb +0 -22
  181. data/lib/sqa/constants.rb +0 -23
  182. data/lib/sqa/indicator/average_true_range.rb +0 -33
  183. data/lib/sqa/indicator/bollinger_bands.rb +0 -28
  184. data/lib/sqa/indicator/candlestick_pattern_recognizer.rb +0 -60
  185. data/lib/sqa/indicator/donchian_channel.rb +0 -29
  186. data/lib/sqa/indicator/double_top_bottom_pattern.rb +0 -34
  187. data/lib/sqa/indicator/elliott_wave_theory.rb +0 -57
  188. data/lib/sqa/indicator/exponential_moving_average.rb +0 -25
  189. data/lib/sqa/indicator/exponential_moving_average_trend.rb +0 -36
  190. data/lib/sqa/indicator/fibonacci_retracement.rb +0 -23
  191. data/lib/sqa/indicator/head_and_shoulders_pattern.rb +0 -26
  192. data/lib/sqa/indicator/market_profile.rb +0 -32
  193. data/lib/sqa/indicator/mean_reversion.rb +0 -37
  194. data/lib/sqa/indicator/momentum.rb +0 -28
  195. data/lib/sqa/indicator/moving_average_convergence_divergence.rb +0 -29
  196. data/lib/sqa/indicator/peaks_and_valleys.rb +0 -29
  197. data/lib/sqa/indicator/predict_next_value.rb +0 -202
  198. data/lib/sqa/indicator/relative_strength_index.rb +0 -47
  199. data/lib/sqa/indicator/simple_moving_average.rb +0 -24
  200. data/lib/sqa/indicator/simple_moving_average_trend.rb +0 -32
  201. data/lib/sqa/indicator/stochastic_oscillator.rb +0 -68
  202. data/lib/sqa/indicator/true_range.rb +0 -39
  203. data/lib/sqa/trade.rb +0 -26
data/CLAUDE.md ADDED
@@ -0,0 +1,674 @@
1
+ # CLAUDE.md
2
+
3
+ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
4
+
5
+ ## Project Overview
6
+
7
+ SQA (Simple Qualitative Analysis) is a Ruby library for stock market technical analysis designed for educational purposes. It provides high-performance data structures, trading strategies, and integrates with the `sqa-tai` gem for 150+ technical indicators.
8
+
9
+ **Important:** SQA is a **library-only gem** with no CLI functionality. The `sqa-console` executable launches an IRB console for interactive experimentation.
10
+
11
+ ## Common Development Commands
12
+
13
+ ### Running Tests
14
+ ```bash
15
+ rake test # Run unit tests with Minitest
16
+ just test # Alternative: Run tests via justfile
17
+ just coverage # Run tests and open coverage report
18
+ ```
19
+
20
+ ### Building and Installing
21
+ ```bash
22
+ rake install # Install gem locally
23
+ just install # Install with TOC update and man page generation
24
+ rake build # Build gem package
25
+ ```
26
+
27
+ ### Code Quality
28
+ ```bash
29
+ just flay # Static code analysis for duplication
30
+ ```
31
+
32
+ ### Documentation
33
+ ```bash
34
+ rake toc # Update README table of contents
35
+ rake man # Generate man pages
36
+ ```
37
+
38
+ ### Interactive Console
39
+ ```bash
40
+ sqa-console # Launch IRB with SQA library loaded
41
+ ```
42
+
43
+ ## Architecture Overview
44
+
45
+ ### Core Module Structure
46
+ - **SQA::Stock**: Primary domain object representing a stock with price history and metadata
47
+ - **SQA::DataFrame**: High-performance wrapper around Polars for time series data manipulation
48
+ - **SQA::DataFrame::Data**: Stock metadata storage (ticker, name, exchange, source, indicators, overview)
49
+ - **SQAI / SQA::TAI**: Access to 150+ technical indicators from `sqa-tai` gem (TA-Lib wrapper)
50
+ - **SQA::Strategy**: Trading strategy framework with 12+ built-in strategies
51
+ - **SQA::Portfolio**: Position and trade tracking with P&L calculations (265 lines)
52
+ - **SQA::Backtest**: Strategy simulation with comprehensive performance metrics (345 lines)
53
+ - **SQA::Ticker**: Stock symbol validation and lookup
54
+ - **SQA::Config**: Configuration management with YAML/TOML support
55
+
56
+ ### Advanced Module Structure
57
+ - **SQA::StrategyGenerator**: Reverse-engineer profitable trades to discover patterns (690 lines, enhanced with Pattern Context)
58
+ - **SQA::GeneticProgram**: Evolutionary algorithm for strategy parameter optimization (259 lines)
59
+ - **SQA::Strategy::KBS**: RETE-based forward-chaining inference engine (454 lines)
60
+ - **SQA::Stream**: Real-time price stream processor with callbacks (343 lines)
61
+ - **SQA::FPOP**: Future Period Loss/Profit analysis utilities (243 lines)
62
+ - **SQA::MarketRegime**: Bull/bear/sideways market detection with volatility analysis (176 lines)
63
+ - **SQA::SeasonalAnalyzer**: Calendar-dependent pattern discovery (monthly/quarterly) (185 lines)
64
+ - **SQA::SectorAnalyzer**: Cross-stock analysis with KBS blackboards per sector (242 lines)
65
+ - **SQA::RiskManager**: Comprehensive risk management with VaR, CVaR, position sizing (566 lines)
66
+ - **SQA::PortfolioOptimizer**: Multi-objective portfolio optimization and rebalancing (389 lines)
67
+ - **SQA::Ensemble**: Strategy combination with voting and meta-learning (358 lines)
68
+ - **SQA::MultiTimeframe**: Multi-timeframe analysis and trend alignment (398 lines)
69
+ - **SQA::PatternMatcher**: Pattern similarity search with forecasting (567 lines)
70
+
71
+ ### Data Flow
72
+
73
+ **Basic Flow:**
74
+ 1. Create `SQA::Stock` with ticker symbol
75
+ 2. Stock fetches data from Alpha Vantage or Yahoo Finance
76
+ 3. Data stored in Polars-based `SQA::DataFrame`
77
+ 4. Apply technical indicators via `SQAI` / `SQA::TAI` (from sqa-tai gem)
78
+ 5. Execute trading strategies to generate buy/sell/hold signals
79
+ 6. Analyze results with statistical functions
80
+
81
+ **Advanced Workflows:**
82
+
83
+ **Backtesting:**
84
+ 1. Load historical data via `SQA::Stock`
85
+ 2. Create `SQA::Backtest` with stock, strategy, and capital
86
+ 3. Backtest simulates trades using `SQA::Portfolio`
87
+ 4. Returns `Results` with metrics: Sharpe ratio, max drawdown, win rate, etc.
88
+
89
+ **Pattern Discovery:**
90
+ 1. `SQA::StrategyGenerator` scans historical data for profitable points
91
+ 2. Identifies indicator states at each profitable entry
92
+ 3. Mines patterns from indicator combinations
93
+ 4. Generates executable strategies from patterns
94
+ 5. Strategies can be backtested or used live
95
+
96
+ **Parameter Optimization:**
97
+ 1. `SQA::GeneticProgram` defines parameter space (genes)
98
+ 2. Creates random population of strategies
99
+ 3. Evaluates fitness via backtesting
100
+ 4. Evolves through selection, crossover, mutation
101
+ 5. Returns best individual with optimal parameters
102
+
103
+ **Real-Time Trading:**
104
+ 1. `SQA::Stream` receives live price updates
105
+ 2. Maintains rolling window of recent data
106
+ 3. Calculates indicators on-the-fly (with caching)
107
+ 4. Executes multiple strategies in parallel
108
+ 5. Aggregates signals and fires callbacks
109
+ 6. Callbacks can execute trades, send alerts, log data
110
+
111
+ **FPL Analysis:**
112
+ 1. `SQA::FPOP.fpl(prices, fpop: 10)` calculates min/max future deltas
113
+ 2. `SQA::FPOP.fpl_analysis(prices, fpop: 10)` adds risk metrics and direction
114
+ 3. `SQA::FPOP.filter_by_quality()` filters by magnitude, risk, direction
115
+ 4. Integration with `StrategyGenerator` via `max_fpl_risk` parameter
116
+ 5. DataFrame convenience methods: `df.fpl()` and `df.fpl_analysis()`
117
+
118
+ **Market Regime Analysis:**
119
+ 1. `SQA::MarketRegime.detect(stock)` classifies current market
120
+ 2. Returns regime type (bull/bear/sideways), volatility, and strength
121
+ 3. `detect_history(stock)` identifies regime changes over time
122
+ 4. `split_by_regime(stock)` groups data by regime periods
123
+ 5. Enables regime-specific strategy selection
124
+
125
+ **Seasonal Pattern Discovery:**
126
+ 1. `SQA::SeasonalAnalyzer.analyze(stock)` finds calendar patterns
127
+ 2. Identifies best/worst months and quarters
128
+ 3. `detect_seasonality()` determines if patterns are significant
129
+ 4. `filter_by_months()` and `filter_by_quarters()` extract seasonal data
130
+ 5. Enables time-of-year pattern validation
131
+
132
+ **Sector Analysis:**
133
+ 1. `SQA::SectorAnalyzer` creates KBS blackboard per sector
134
+ 2. Stocks registered with sector classification
135
+ 3. `discover_sector_patterns()` finds cross-stock patterns
136
+ 4. `detect_sector_regime()` gets consensus market view
137
+ 5. Persistent SQLite storage for each sector's knowledge base
138
+
139
+ **Context-Aware Pattern Discovery:**
140
+ 1. `StrategyGenerator.discover_context_aware_patterns()` adds metadata
141
+ 2. Patterns tagged with market regime, valid months/quarters, sector
142
+ 3. `PatternContext.valid_for?(date, regime, sector)` runtime validation
143
+ 4. `walk_forward_validate()` prevents overfitting with out-of-sample testing
144
+ 5. Patterns know when they should and shouldn't be used
145
+
146
+ **Risk Management:**
147
+ 1. `SQA::RiskManager.var(returns, confidence: 0.95)` calculates Value at Risk
148
+ 2. `SQA::RiskManager.cvar(returns)` for Conditional VaR (Expected Shortfall)
149
+ 3. Position sizing: `kelly_criterion()`, `fixed_fractional()`, `percent_volatility()`
150
+ 4. Risk metrics: `sharpe_ratio()`, `sortino_ratio()`, `calmar_ratio()`
151
+ 5. `max_drawdown()` and `monte_carlo_simulation()` for risk assessment
152
+
153
+ **Portfolio Optimization:**
154
+ 1. `SQA::PortfolioOptimizer.maximum_sharpe(returns_matrix)` finds optimal weights
155
+ 2. `minimum_variance()` and `risk_parity()` for conservative allocations
156
+ 3. `efficient_frontier()` generates risk/return curve
157
+ 4. `multi_objective()` optimizes multiple goals simultaneously
158
+ 5. `rebalance()` calculates trades needed to reach target allocation
159
+
160
+ **Ensemble Strategies:**
161
+ 1. `SQA::Ensemble.new(strategies: [...], voting_method: :majority)` combines strategies
162
+ 2. Voting methods: `:majority`, `:weighted`, `:unanimous`, `:confidence`
163
+ 3. `update_weight()` adjusts based on performance
164
+ 4. `rotate()` selects best strategy for current market conditions
165
+ 5. `backtest_comparison()` evaluates ensemble vs individuals
166
+
167
+ **Multi-Timeframe Analysis:**
168
+ 1. `SQA::MultiTimeframe.new(stock: stock)` converts daily → weekly → monthly
169
+ 2. `trend_alignment()` checks if all timeframes agree
170
+ 3. `signal()` combines higher timeframe trend with lower timeframe timing
171
+ 4. `support_resistance()` finds levels that appear across multiple timeframes
172
+ 5. `detect_divergence()` identifies price/indicator disagreement
173
+
174
+ **Pattern Similarity Search:**
175
+ 1. `SQA::PatternMatcher.find_similar(lookback: 10)` finds historical matches
176
+ 2. Methods: `:euclidean`, `:dtw` (Dynamic Time Warping), `:correlation`
177
+ 3. `forecast()` predicts future moves based on similar past patterns
178
+ 4. `detect_chart_pattern()` finds double tops/bottoms, head & shoulders, triangles
179
+ 5. `cluster_patterns()` groups similar patterns with k-means
180
+
181
+ ### Key Design Patterns
182
+ - **Plugin Architecture**: Strategies are pluggable modules
183
+ - **Data Source Abstraction**: Multiple data providers (Alpha Vantage, Yahoo Finance) with common interface
184
+ - **Delegation Pattern**: DataFrame delegates to Polars for high-performance operations
185
+ - **Configuration Hierarchy**: defaults < environment variables < config file
186
+
187
+ ## Important Implementation Notes
188
+
189
+ ### Data Sources
190
+ - **Alpha Vantage API** requires `AV_API_KEY` or `ALPHAVANTAGE_API_KEY` environment variable
191
+ - **Yahoo Finance** scraping available as fallback (no API, less reliable)
192
+ - CSV file imports supported for historical data (place in data directory as `ticker.csv`)
193
+
194
+ ### Configuration
195
+ - Config files: YAML or TOML in `~/.sqa.*`
196
+ - Data directory: `~/sqa_data/` (default, configurable)
197
+ - Environment variables:
198
+ - `AV_API_KEY` or `ALPHAVANTAGE_API_KEY` - Alpha Vantage API key
199
+ - Custom config via `SQA::Config.new(data_dir: '...')`
200
+
201
+ ### DataFrame Implementation
202
+ - Uses `polars-df` gem (Rust-backed, blazingly fast)
203
+ - Wraps Polars::DataFrame with convenience methods
204
+ - Custom statistics via `lite-statistics` gem monkey patches on Arrays
205
+ - **Always** prefer column-based operations over row iterations for performance
206
+ - Access underlying Polars DataFrame via `.data` attribute
207
+
208
+ ### Technical Indicators
209
+ - **All indicators** provided by separate `sqa-tai` gem
210
+ - `sqa-tai` wraps TA-Lib C library (industry standard)
211
+ - Access via `SQAI.indicator_name(prices, options)` or `SQA::TAI.indicator_name(...)`
212
+ - 150+ indicators available: SMA, EMA, RSI, MACD, Bollinger Bands, ADX, ATR, etc.
213
+ - See: https://github.com/MadBomber/sqa-tai
214
+
215
+ ### Data Ordering (CRITICAL)
216
+ **TA-Lib Requirement**: Arrays MUST be in **OLDEST-FIRST** (ascending chronological) order
217
+ - Index [0] = oldest data point
218
+ - Index [last] = newest/most recent data point
219
+ - This applies to ALL TA-Lib indicators
220
+
221
+ **SQA Implementation**:
222
+ - ✅ CSV files stored in **ASCENDING order** (oldest-first) for TA-Lib compatibility
223
+ - ✅ DataFrames maintain ascending order after updates
224
+ - ✅ `.to_a` extracts arrays ready for TA-Lib (no reversal needed)
225
+ - ✅ `concat_and_deduplicate!` sorts ascending by default
226
+ - ⚠️ Alpha Vantage API returns data newest-first, but SQA automatically sorts to ascending
227
+
228
+ **Example**:
229
+ ```ruby
230
+ prices = stock.df["adj_close_price"].to_a
231
+ # => [100.0, 101.5, 103.2, ...] (oldest to newest - ready for TA-Lib)
232
+
233
+ rsi = SQAI.rsi(prices, period: 14) # Correct order, no reversal needed
234
+ ```
235
+
236
+ ### Testing Approach
237
+ - Minitest framework in `/test/` directory
238
+ - SimpleCov for coverage reporting
239
+ - Test data fixtures in `test/test_helper.rb`
240
+ - **Note:** Indicator tests may need updates after migrating to sqa-tai (different return values)
241
+
242
+ ## Development Guidelines
243
+
244
+ ### When Adding New Strategies
245
+ 1. Create new file in `lib/sqa/strategy/`
246
+ 2. Define class under `SQA::Strategy::` namespace
247
+ 3. Implement `self.trade(vector)` class method
248
+ 4. Return `:buy`, `:sell`, or `:hold`
249
+ 5. Add corresponding test in `test/strategy/`
250
+
251
+ **Example:**
252
+ ```ruby
253
+ class SQA::Strategy::MyStrategy
254
+ def self.trade(vector)
255
+ # vector is an OpenStruct with indicator values
256
+ if vector.rsi < 30
257
+ :buy
258
+ elsif vector.rsi > 70
259
+ :sell
260
+ else
261
+ :hold
262
+ end
263
+ end
264
+ end
265
+ ```
266
+
267
+ ### Working with DataFrames
268
+ - Use Polars native operations when possible (accessed via `df.data`)
269
+ - Avoid Ruby loops over rows for performance
270
+ - Leverage vectorized operations
271
+ - Example: `df.data["close_price"]` returns Polars::Series
272
+
273
+ ### Working with Stock Metadata (DataFrame::Data)
274
+ - **Purpose**: Stores stock metadata separate from price/volume DataFrame
275
+ - **Attributes**: ticker, name, exchange, source, indicators, overview
276
+ - **Dual initialization**:
277
+ - From hash: `SQA::DataFrame::Data.new(JSON.parse(json_string))`
278
+ - From keywords: `SQA::DataFrame::Data.new(ticker: 'AAPL', source: :alpha_vantage, indicators: {})`
279
+ - **JSON serialization**: `data.to_json` for persistence
280
+ - **Used by**: `SQA::Stock` to persist metadata in `~/sqa_data/ticker.json`
281
+ - **File location**: `lib/sqa/data_frame/data.rb`
282
+
283
+ ### Working with Indicators
284
+ - **Do NOT add indicators to SQA** - contribute to `sqa-tai` gem instead
285
+ - Use indicators: `SQAI.sma(prices_array, period: 20)`
286
+ - All indicators work on Ruby Arrays, not DataFrames
287
+ - Extract price array first: `prices = stock.df["close_price"].to_a`
288
+
289
+ ### API Integration
290
+ - New data sources go in `lib/sqa/data_frame/`
291
+ - Follow existing adapter pattern (see `alpha_vantage.rb` and `yahoo_finance.rb`)
292
+ - Handle rate limiting and errors gracefully
293
+ - Return Polars-compatible data structures
294
+
295
+ ### Adding New Data Sources
296
+ 1. Create `lib/sqa/data_frame/my_source.rb`
297
+ 2. Define class `SQA::DataFrame::MySource`
298
+ 3. Implement `self.recent(ticker, **options)` method
299
+ 4. Return data in format compatible with Polars::DataFrame
300
+ 5. Add mapping for column names if needed
301
+
302
+ ## Critical Constraints
303
+
304
+ - This is an **EDUCATIONAL tool** - maintain clear disclaimers
305
+ - Do NOT remove financial risk warnings from README or docs
306
+ - **No CLI** - this is a library, not a command-line application
307
+ - Performance matters - prefer vectorized DataFrame operations
308
+ - Maintain backward compatibility with existing data file formats
309
+ - API keys from environment variables only (no api_key_manager)
310
+
311
+ ### Testing Guidelines
312
+ - All new features should have corresponding tests in `/test/`
313
+ - Integration tests require `RUN_INTEGRATION_TESTS=1` environment variable
314
+ - Use `skip` for tests requiring network access or long runtimes
315
+ - Portfolio, Backtest, Stream have comprehensive test coverage
316
+ - GP and StrategyGenerator tests available but skipped by default (long running)
317
+
318
+ ## File Structure
319
+
320
+ ```
321
+ lib/
322
+ ├── api/
323
+ │ └── alpha_vantage_api.rb # Alpha Vantage API client (462 lines)
324
+ ├── patches/
325
+ │ └── string.rb # String helpers (camelize, constantize, underscore)
326
+ └── sqa/
327
+ ├── backtest.rb # ✨ NEW: Backtesting framework (345 lines)
328
+ ├── config.rb # Configuration management
329
+ ├── data_frame.rb # Polars DataFrame wrapper
330
+ ├── data_frame/
331
+ │ ├── alpha_vantage.rb # Alpha Vantage data adapter
332
+ │ ├── data.rb # Stock metadata storage class (93 lines)
333
+ │ └── yahoo_finance.rb # Yahoo Finance scraper
334
+ ├── errors.rb # Error classes
335
+ ├── ensemble.rb # ✨ NEW: Strategy combination and voting (358 lines)
336
+ ├── fpop.rb # ✨ NEW: Future Period Loss/Profit analysis (243 lines)
337
+ ├── gp.rb # ✨ NEW: Genetic programming (259 lines, COMPLETE)
338
+ ├── indicator.rb # Delegates to sqa-tai gem
339
+ ├── init.rb # Module initialization
340
+ ├── market_regime.rb # ✨ NEW: Market regime detection (176 lines)
341
+ ├── multi_timeframe.rb # ✨ NEW: Multi-timeframe analysis (398 lines)
342
+ ├── pattern_matcher.rb # ✨ NEW: Pattern similarity search (567 lines)
343
+ ├── portfolio.rb # ✨ NEW: Portfolio management (265 lines, COMPLETE)
344
+ ├── portfolio_optimizer.rb # ✨ NEW: Portfolio optimization (389 lines)
345
+ ├── risk_manager.rb # ✨ NEW: Risk management (566 lines)
346
+ ├── seasonal_analyzer.rb # ✨ NEW: Seasonal pattern discovery (185 lines)
347
+ ├── sector_analyzer.rb # ✨ NEW: Sector analysis with KBS (242 lines)
348
+ ├── stock.rb # Stock class with data management
349
+ ├── strategy.rb # Strategy framework
350
+ ├── strategy/
351
+ │ ├── bollinger_bands.rb # ✨ NEW: Bollinger Bands strategy
352
+ │ ├── common.rb # Shared strategy utilities
353
+ │ ├── consensus.rb # Consensus from multiple strategies
354
+ │ ├── ema.rb # EMA-based strategy
355
+ │ ├── kbs_strategy.rb # ✨ NEW: RETE-based KBS (454 lines)
356
+ │ ├── macd.rb # ✨ NEW: MACD crossover strategy
357
+ │ ├── mp.rb # Market Profile strategy
358
+ │ ├── mr.rb # Mean Reversion strategy
359
+ │ ├── random.rb # Random signals (testing)
360
+ │ ├── rsi.rb # RSI-based strategy
361
+ │ ├── sma.rb # SMA-based strategy
362
+ │ ├── stochastic.rb # ✨ NEW: Stochastic oscillator strategy
363
+ │ └── volume_breakout.rb # ✨ NEW: Volume breakout strategy
364
+ ├── stream.rb # ✨ NEW: Real-time price streaming (343 lines)
365
+ ├── strategy_generator.rb # ✨ NEW: Pattern discovery (690 lines)
366
+ ├── ticker.rb # Ticker validation
367
+ └── version.rb # Version constant
368
+
369
+ examples/
370
+ ├── README.md # ✨ NEW: Comprehensive examples guide
371
+ ├── advanced_features_example.rb # ✨ NEW: All advanced features demo (396 lines)
372
+ ├── fpop_analysis_example.rb # ✨ NEW: FPL analysis utilities (191 lines)
373
+ ├── genetic_programming_example.rb # ✨ NEW: GP parameter evolution
374
+ ├── kbs_strategy_example.rb # ✨ NEW: RETE rule-based trading
375
+ ├── pattern_context_example.rb # ✨ NEW: Context-aware patterns (280 lines)
376
+ ├── realtime_stream_example.rb # ✨ NEW: Live price processing
377
+ └── strategy_generator_example.rb # ✨ NEW: Pattern mining
378
+
379
+ test/
380
+ ├── backtest_test.rb # ✨ NEW: Backtest tests
381
+ ├── ensemble_test.rb # ✨ NEW: Ensemble methods tests (109 lines, 12 tests)
382
+ ├── fpop_test.rb # ✨ NEW: FPL analysis tests (154 lines)
383
+ ├── gp_test.rb # ✨ NEW: Genetic programming tests
384
+ ├── market_regime_test.rb # ✨ NEW: Market regime tests (165 lines)
385
+ ├── multi_timeframe_test.rb # ✨ NEW: Multi-timeframe tests (77 lines, 7 tests)
386
+ ├── pattern_context_test.rb # ✨ NEW: Pattern context tests (177 lines)
387
+ ├── pattern_context_integration_test.rb # ✨ NEW: Integration tests (342 lines)
388
+ ├── pattern_matcher_test.rb # ✨ NEW: Pattern matching tests (155 lines, 15 tests)
389
+ ├── portfolio_test.rb # ✨ NEW: Portfolio tests
390
+ ├── portfolio_optimizer_test.rb # ✨ NEW: Portfolio optimization tests (122 lines, 10 tests)
391
+ ├── risk_manager_test.rb # ✨ NEW: Risk management tests (183 lines, 22 tests)
392
+ ├── seasonal_analyzer_test.rb # ✨ NEW: Seasonal analysis tests (203 lines)
393
+ ├── sector_analyzer_test.rb # ✨ NEW: Sector analysis tests (162 lines)
394
+ ├── stream_test.rb # ✨ NEW: Stream processor tests
395
+ ├── strategy_generator_test.rb # ✨ NEW: Strategy generator tests
396
+ ├── data_frame_test.rb # DataFrame tests
397
+ ├── test_helper.rb # Test configuration
398
+ └── indicator/ # Indicator tests (legacy, may need updates)
399
+ ```
400
+
401
+ **Line Counts:**
402
+ - Core: ~2,000 lines (DataFrame, Stock, Strategy framework)
403
+ - Advanced Features: ~5,428 lines (13 advanced modules)
404
+ * Pattern Context System: 1,089 lines (FPOP, MarketRegime, SeasonalAnalyzer, SectorAnalyzer)
405
+ * Trading Infrastructure: 867 lines (Portfolio, Backtest, Stream)
406
+ * Intelligence: 1,403 lines (GP, KBS, StrategyGenerator)
407
+ * New Advanced Features: 2,278 lines (RiskManager, PortfolioOptimizer, Ensemble, MultiTimeframe, PatternMatcher)
408
+ - Tests: ~2,650 lines (17 test files, 132+ tests)
409
+ - Examples: ~2,250 lines (8 comprehensive examples)
410
+ - Total: ~12,328 lines
411
+
412
+ ## Common Gotchas
413
+
414
+ 1. **DataFrame vs Polars**: `df` is SQA::DataFrame, `df.data` is Polars::DataFrame
415
+ 2. **Indicators need Arrays**: Extract data with `.to_a` before passing to indicators
416
+ 3. **No CLI commands**: Previous CLI functionality has been removed
417
+ 4. **Indicators in separate gem**: Technical indicators are in `sqa-tai`, not SQA
418
+ 5. **API key format changed**: Use `AV_API_KEY` not `AV_API_KEYS` (singular)
419
+ 6. **Strategies need OpenStruct**: Pass data to strategies as OpenStruct with named fields
420
+
421
+ ## Advanced Features Quick Reference
422
+
423
+ ### Portfolio Management
424
+ ```ruby
425
+ portfolio = SQA::Portfolio.new(initial_cash: 10_000, commission: 1.0)
426
+ portfolio.buy('AAPL', shares: 10, price: 150.0)
427
+ portfolio.sell('AAPL', shares: 5, price: 160.0)
428
+ portfolio.value({ 'AAPL' => 165.0 })
429
+ ```
430
+
431
+ ### Backtesting
432
+ ```ruby
433
+ backtest = SQA::Backtest.new(stock: stock, strategy: SQA::Strategy::RSI)
434
+ results = backtest.run
435
+ puts "Return: #{results.total_return}%, Sharpe: #{results.sharpe_ratio}"
436
+ ```
437
+
438
+ ### Strategy Generator
439
+ ```ruby
440
+ gen = SQA::StrategyGenerator.new(stock: stock, min_gain_percent: 10.0)
441
+ patterns = gen.discover_patterns
442
+ strategy = gen.generate_strategy(pattern_index: 0)
443
+ ```
444
+
445
+ ### Genetic Programming
446
+ ```ruby
447
+ gp = SQA::GeneticProgram.new(stock: stock, population_size: 50)
448
+ gp.define_genes(rsi_period: (7..30).to_a)
449
+ gp.fitness { |genes| backtest_with(genes).total_return }
450
+ best = gp.evolve
451
+ ```
452
+
453
+ ### Knowledge-Based Strategy
454
+ ```ruby
455
+ strategy = SQA::Strategy::KBS.new(load_defaults: false)
456
+ strategy.add_rule :my_rule do
457
+ on :rsi, { level: :oversold }
458
+ on :macd, { crossover: :bullish }
459
+ perform { assert(:signal, { action: :buy }) }
460
+ end
461
+ ```
462
+
463
+ ### Real-Time Streaming
464
+ ```ruby
465
+ stream = SQA::Stream.new(ticker: 'AAPL', strategies: [SQA::Strategy::RSI])
466
+ stream.on_signal { |signal, data| execute_trade(signal, data) }
467
+ stream.update(price: 150.25, volume: 1_000_000)
468
+ ```
469
+
470
+ ### FPL Analysis
471
+ ```ruby
472
+ # Basic FPL calculation
473
+ prices = stock.df["adj_close_price"].to_a
474
+ fpl_data = SQA::FPOP.fpl(prices, fpop: 10)
475
+ # => [[min_delta, max_delta], ...]
476
+
477
+ # Comprehensive analysis
478
+ analysis = SQA::FPOP.fpl_analysis(prices, fpop: 10)
479
+ puts "Risk: #{analysis[:risk]}%, Direction: #{analysis[:direction]}"
480
+
481
+ # Filter high-quality opportunities
482
+ filtered = SQA::FPOP.filter_by_quality(
483
+ analysis,
484
+ min_magnitude: 5.0,
485
+ max_risk: 25.0,
486
+ directions: [:UP]
487
+ )
488
+ ```
489
+
490
+ ### Market Regime Detection
491
+ ```ruby
492
+ # Detect current regime
493
+ regime = SQA::MarketRegime.detect(stock)
494
+ puts "Regime: #{regime[:type]}" # => :bull, :bear, or :sideways
495
+ puts "Volatility: #{regime[:volatility]}" # => :low, :medium, :high
496
+
497
+ # Get regime history
498
+ history = SQA::MarketRegime.detect_history(stock, window: 60)
499
+
500
+ # Split data by regime
501
+ splits = SQA::MarketRegime.split_by_regime(stock)
502
+ bull_periods = splits[:bull]
503
+ ```
504
+
505
+ ### Seasonal Analysis
506
+ ```ruby
507
+ # Analyze seasonal patterns
508
+ seasonal = SQA::SeasonalAnalyzer.analyze(stock)
509
+ puts "Best months: #{seasonal[:best_months]}" # => [10, 11, 12]
510
+ puts "Best quarters: #{seasonal[:best_quarters]}" # => [4, 1]
511
+
512
+ # Filter for Q4 only
513
+ q4_data = SQA::SeasonalAnalyzer.filter_by_quarters(stock, [4])
514
+ ```
515
+
516
+ ### Sector Analysis
517
+ ```ruby
518
+ analyzer = SQA::SectorAnalyzer.new
519
+ analyzer.add_stock('AAPL', sector: :technology)
520
+ analyzer.add_stock('MSFT', sector: :technology)
521
+
522
+ # Detect sector regime
523
+ tech_stocks = ['AAPL', 'MSFT'].map { |t| SQA::Stock.new(ticker: t) }
524
+ regime = analyzer.detect_sector_regime(:technology, tech_stocks)
525
+
526
+ # Find sector-wide patterns
527
+ patterns = analyzer.discover_sector_patterns(:technology, tech_stocks)
528
+ ```
529
+
530
+ ### Context-Aware Pattern Discovery
531
+ ```ruby
532
+ # Discover patterns with context
533
+ generator = SQA::StrategyGenerator.new(stock: stock, fpop: 10)
534
+ patterns = generator.discover_context_aware_patterns(
535
+ analyze_regime: true,
536
+ analyze_seasonal: true,
537
+ sector: :technology
538
+ )
539
+
540
+ # Walk-forward validation
541
+ validated = generator.walk_forward_validate(
542
+ train_size: 250,
543
+ test_size: 60
544
+ )
545
+
546
+ # Runtime validation
547
+ pattern = patterns.first
548
+ valid = pattern.context.valid_for?(
549
+ date: Date.today,
550
+ regime: :bull,
551
+ sector: :technology
552
+ )
553
+ ```
554
+
555
+ ### Risk Management
556
+ ```ruby
557
+ # Value at Risk
558
+ returns = prices.each_cons(2).map { |a, b| (b - a) / a }
559
+ var_95 = SQA::RiskManager.var(returns, confidence: 0.95, method: :historical)
560
+ cvar_95 = SQA::RiskManager.cvar(returns, confidence: 0.95)
561
+
562
+ # Position sizing
563
+ position = SQA::RiskManager.kelly_criterion(
564
+ win_rate: 0.60,
565
+ avg_win: 0.10,
566
+ avg_loss: 0.05,
567
+ capital: 10_000
568
+ )
569
+
570
+ # Risk metrics
571
+ sharpe = SQA::RiskManager.sharpe_ratio(returns)
572
+ sortino = SQA::RiskManager.sortino_ratio(returns)
573
+ dd = SQA::RiskManager.max_drawdown(prices)
574
+ ```
575
+
576
+ ### Portfolio Optimization
577
+ ```ruby
578
+ # Multiple stocks returns
579
+ returns_matrix = stocks.map do |stock|
580
+ stock.df["adj_close_price"].to_a.each_cons(2).map { |a, b| (b - a) / a }
581
+ end
582
+
583
+ # Maximum Sharpe
584
+ max_sharpe = SQA::PortfolioOptimizer.maximum_sharpe(returns_matrix)
585
+ # => { weights: [0.4, 0.3, 0.3], sharpe: 1.5, return: 0.12, volatility: 0.15 }
586
+
587
+ # Minimum variance
588
+ min_var = SQA::PortfolioOptimizer.minimum_variance(returns_matrix)
589
+
590
+ # Multi-objective
591
+ multi = SQA::PortfolioOptimizer.multi_objective(
592
+ returns_matrix,
593
+ objectives: { maximize_return: 0.4, minimize_volatility: 0.3, minimize_drawdown: 0.3 }
594
+ )
595
+ ```
596
+
597
+ ### Ensemble Strategies
598
+ ```ruby
599
+ # Create ensemble
600
+ ensemble = SQA::Ensemble.new(
601
+ strategies: [SQA::Strategy::RSI, SQA::Strategy::MACD],
602
+ voting_method: :majority
603
+ )
604
+
605
+ # Get ensemble signal
606
+ signal = ensemble.signal(vector) # => :buy, :sell, or :hold
607
+
608
+ # Rotate based on market
609
+ selected = ensemble.rotate(stock)
610
+ ```
611
+
612
+ ### Multi-Timeframe Analysis
613
+ ```ruby
614
+ # Create analyzer
615
+ mta = SQA::MultiTimeframe.new(stock: stock)
616
+
617
+ # Check trend alignment
618
+ alignment = mta.trend_alignment
619
+ # => { daily: :up, weekly: :up, monthly: :up, aligned: true, direction: :bullish }
620
+
621
+ # Multi-timeframe signal
622
+ signal = mta.signal(
623
+ strategy_class: SQA::Strategy::RSI,
624
+ higher_timeframe: :weekly,
625
+ lower_timeframe: :daily
626
+ )
627
+
628
+ # Find strong support/resistance
629
+ levels = mta.support_resistance(tolerance: 0.02)
630
+ ```
631
+
632
+ ### Pattern Similarity Search
633
+ ```ruby
634
+ # Create matcher
635
+ matcher = SQA::PatternMatcher.new(stock: stock)
636
+
637
+ # Find similar patterns
638
+ similar = matcher.find_similar(lookback: 10, num_matches: 5, method: :euclidean)
639
+ # => [{ distance: 0.05, future_return: 0.12, pattern: [...] }, ...]
640
+
641
+ # Forecast based on similar patterns
642
+ forecast = matcher.forecast(lookback: 10, forecast_periods: 5)
643
+ # => { forecast_price: 155.0, forecast_return: 0.03, confidence_interval_95: [150, 160] }
644
+
645
+ # Detect chart patterns
646
+ patterns = matcher.detect_chart_pattern(:double_top)
647
+ ```
648
+
649
+ ## Quick Reference
650
+
651
+ ```ruby
652
+ require 'sqa'
653
+
654
+ # Initialize
655
+ SQA.init
656
+
657
+ # Load stock data
658
+ stock = SQA::Stock.new(ticker: 'AAPL')
659
+
660
+ # Get price array
661
+ prices = stock.df["adj_close_price"].to_a
662
+
663
+ # Calculate indicators
664
+ sma = SQAI.sma(prices, period: 20)
665
+ rsi = SQAI.rsi(prices, period: 14)
666
+
667
+ # Execute strategies
668
+ require 'ostruct'
669
+ vector = OpenStruct.new(rsi: rsi.last, prices: prices)
670
+
671
+ strategy = SQA::Strategy.new
672
+ strategy.add SQA::Strategy::RSI
673
+ signals = strategy.execute(vector) # => [:buy, :sell, :hold]
674
+ ```