sqa 0.0.24 → 0.0.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. checksums.yaml +4 -4
  2. data/.goose/memory/development.txt +3 -0
  3. data/.semver +6 -0
  4. data/ARCHITECTURE.md +648 -0
  5. data/CHANGELOG.md +82 -0
  6. data/CLAUDE.md +653 -0
  7. data/COMMITS.md +196 -0
  8. data/DATAFRAME_ARCHITECTURE_REVIEW.md +421 -0
  9. data/NEXT-STEPS.md +154 -0
  10. data/README.md +812 -262
  11. data/TASKS.md +358 -0
  12. data/TEST_RESULTS.md +140 -0
  13. data/TODO.md +42 -0
  14. data/_notes.txt +25 -0
  15. data/bin/sqa-console +11 -0
  16. data/data/talk_talk.json +103284 -0
  17. data/develop_summary.md +313 -0
  18. data/docs/advanced/backtesting.md +206 -0
  19. data/docs/advanced/ensemble.md +68 -0
  20. data/docs/advanced/fpop.md +153 -0
  21. data/docs/advanced/index.md +112 -0
  22. data/docs/advanced/multi-timeframe.md +67 -0
  23. data/docs/advanced/pattern-matcher.md +75 -0
  24. data/docs/advanced/portfolio-optimizer.md +79 -0
  25. data/docs/advanced/portfolio.md +166 -0
  26. data/docs/advanced/risk-management.md +210 -0
  27. data/docs/advanced/strategy-generator.md +158 -0
  28. data/docs/advanced/streaming.md +209 -0
  29. data/docs/ai_and_ml.md +80 -0
  30. data/docs/api/dataframe.md +1115 -0
  31. data/docs/api/index.md +126 -0
  32. data/docs/assets/css/custom.css +88 -0
  33. data/docs/assets/js/mathjax.js +18 -0
  34. data/docs/concepts/index.md +68 -0
  35. data/docs/contributing/index.md +60 -0
  36. data/docs/data-sources/index.md +66 -0
  37. data/docs/data_frame.md +317 -97
  38. data/docs/factors_that_impact_price.md +26 -0
  39. data/docs/finviz.md +11 -0
  40. data/docs/fx_pro_bit.md +25 -0
  41. data/docs/genetic_programming.md +104 -0
  42. data/docs/getting-started/index.md +123 -0
  43. data/docs/getting-started/installation.md +229 -0
  44. data/docs/getting-started/quick-start.md +244 -0
  45. data/docs/i_gotta_an_idea.md +22 -0
  46. data/docs/index.md +163 -0
  47. data/docs/indicators/index.md +97 -0
  48. data/docs/indicators.md +110 -24
  49. data/docs/options.md +8 -0
  50. data/docs/strategies/bollinger-bands.md +146 -0
  51. data/docs/strategies/consensus.md +64 -0
  52. data/docs/strategies/custom.md +310 -0
  53. data/docs/strategies/ema.md +53 -0
  54. data/docs/strategies/index.md +92 -0
  55. data/docs/strategies/kbs.md +164 -0
  56. data/docs/strategies/macd.md +96 -0
  57. data/docs/strategies/market-profile.md +54 -0
  58. data/docs/strategies/mean-reversion.md +58 -0
  59. data/docs/strategies/rsi.md +95 -0
  60. data/docs/strategies/sma.md +55 -0
  61. data/docs/strategies/stochastic.md +63 -0
  62. data/docs/strategies/volume-breakout.md +54 -0
  63. data/docs/tags.md +7 -0
  64. data/docs/true_strength_index.md +46 -0
  65. data/docs/weighted_moving_average.md +48 -0
  66. data/examples/README.md +354 -0
  67. data/examples/advanced_features_example.rb +350 -0
  68. data/examples/fpop_analysis_example.rb +191 -0
  69. data/examples/genetic_programming_example.rb +148 -0
  70. data/examples/kbs_strategy_example.rb +208 -0
  71. data/examples/pattern_context_example.rb +300 -0
  72. data/examples/rails_app/Gemfile +34 -0
  73. data/examples/rails_app/README.md +416 -0
  74. data/examples/rails_app/app/assets/javascripts/application.js +107 -0
  75. data/examples/rails_app/app/assets/stylesheets/application.css +659 -0
  76. data/examples/rails_app/app/controllers/analysis_controller.rb +11 -0
  77. data/examples/rails_app/app/controllers/api/v1/stocks_controller.rb +227 -0
  78. data/examples/rails_app/app/controllers/application_controller.rb +22 -0
  79. data/examples/rails_app/app/controllers/backtest_controller.rb +11 -0
  80. data/examples/rails_app/app/controllers/dashboard_controller.rb +21 -0
  81. data/examples/rails_app/app/controllers/portfolio_controller.rb +7 -0
  82. data/examples/rails_app/app/views/analysis/show.html.erb +209 -0
  83. data/examples/rails_app/app/views/backtest/show.html.erb +171 -0
  84. data/examples/rails_app/app/views/dashboard/index.html.erb +118 -0
  85. data/examples/rails_app/app/views/dashboard/show.html.erb +408 -0
  86. data/examples/rails_app/app/views/errors/show.html.erb +17 -0
  87. data/examples/rails_app/app/views/layouts/application.html.erb +60 -0
  88. data/examples/rails_app/app/views/portfolio/index.html.erb +33 -0
  89. data/examples/rails_app/bin/rails +6 -0
  90. data/examples/rails_app/config/application.rb +45 -0
  91. data/examples/rails_app/config/boot.rb +5 -0
  92. data/examples/rails_app/config/database.yml +18 -0
  93. data/examples/rails_app/config/environment.rb +11 -0
  94. data/examples/rails_app/config/routes.rb +26 -0
  95. data/examples/rails_app/config.ru +8 -0
  96. data/examples/realtime_stream_example.rb +274 -0
  97. data/examples/sinatra_app/Gemfile +22 -0
  98. data/examples/sinatra_app/QUICKSTART.md +159 -0
  99. data/examples/sinatra_app/README.md +461 -0
  100. data/examples/sinatra_app/app.rb +344 -0
  101. data/examples/sinatra_app/config.ru +5 -0
  102. data/examples/sinatra_app/public/css/style.css +659 -0
  103. data/examples/sinatra_app/public/js/app.js +107 -0
  104. data/examples/sinatra_app/views/analyze.erb +306 -0
  105. data/examples/sinatra_app/views/backtest.erb +325 -0
  106. data/examples/sinatra_app/views/dashboard.erb +419 -0
  107. data/examples/sinatra_app/views/error.erb +58 -0
  108. data/examples/sinatra_app/views/index.erb +118 -0
  109. data/examples/sinatra_app/views/layout.erb +61 -0
  110. data/examples/sinatra_app/views/portfolio.erb +43 -0
  111. data/examples/strategy_generator_example.rb +346 -0
  112. data/hsa_portfolio.csv +11 -0
  113. data/justfile +0 -0
  114. data/lib/api/alpha_vantage_api.rb +462 -0
  115. data/lib/sqa/backtest.rb +329 -0
  116. data/lib/sqa/data_frame/alpha_vantage.rb +43 -65
  117. data/lib/sqa/data_frame/data.rb +92 -0
  118. data/lib/sqa/data_frame/yahoo_finance.rb +35 -43
  119. data/lib/sqa/data_frame.rb +148 -243
  120. data/lib/sqa/ensemble.rb +359 -0
  121. data/lib/sqa/fpop.rb +199 -0
  122. data/lib/sqa/gp.rb +259 -0
  123. data/lib/sqa/indicator.rb +5 -8
  124. data/lib/sqa/init.rb +15 -8
  125. data/lib/sqa/market_regime.rb +240 -0
  126. data/lib/sqa/multi_timeframe.rb +379 -0
  127. data/lib/sqa/pattern_matcher.rb +497 -0
  128. data/lib/sqa/portfolio.rb +260 -6
  129. data/lib/sqa/portfolio_optimizer.rb +377 -0
  130. data/lib/sqa/risk_manager.rb +442 -0
  131. data/lib/sqa/seasonal_analyzer.rb +209 -0
  132. data/lib/sqa/sector_analyzer.rb +300 -0
  133. data/lib/sqa/stock.rb +67 -125
  134. data/lib/sqa/strategy/bollinger_bands.rb +42 -0
  135. data/lib/sqa/strategy/consensus.rb +5 -2
  136. data/lib/sqa/strategy/kbs_strategy.rb +470 -0
  137. data/lib/sqa/strategy/macd.rb +46 -0
  138. data/lib/sqa/strategy/mp.rb +1 -1
  139. data/lib/sqa/strategy/stochastic.rb +60 -0
  140. data/lib/sqa/strategy/volume_breakout.rb +57 -0
  141. data/lib/sqa/strategy.rb +5 -0
  142. data/lib/sqa/strategy_generator.rb +947 -0
  143. data/lib/sqa/stream.rb +361 -0
  144. data/lib/sqa/version.rb +1 -7
  145. data/lib/sqa.rb +23 -16
  146. data/main.just +81 -0
  147. data/mkdocs.yml +288 -0
  148. data/trace.log +0 -0
  149. metadata +261 -51
  150. data/bin/sqa +0 -6
  151. data/lib/patches/dry-cli.rb +0 -228
  152. data/lib/sqa/activity.rb +0 -10
  153. data/lib/sqa/cli.rb +0 -62
  154. data/lib/sqa/commands/analysis.rb +0 -309
  155. data/lib/sqa/commands/base.rb +0 -139
  156. data/lib/sqa/commands/web.rb +0 -199
  157. data/lib/sqa/commands.rb +0 -22
  158. data/lib/sqa/constants.rb +0 -23
  159. data/lib/sqa/indicator/average_true_range.rb +0 -33
  160. data/lib/sqa/indicator/bollinger_bands.rb +0 -28
  161. data/lib/sqa/indicator/candlestick_pattern_recognizer.rb +0 -60
  162. data/lib/sqa/indicator/donchian_channel.rb +0 -29
  163. data/lib/sqa/indicator/double_top_bottom_pattern.rb +0 -34
  164. data/lib/sqa/indicator/elliott_wave_theory.rb +0 -57
  165. data/lib/sqa/indicator/exponential_moving_average.rb +0 -25
  166. data/lib/sqa/indicator/exponential_moving_average_trend.rb +0 -36
  167. data/lib/sqa/indicator/fibonacci_retracement.rb +0 -23
  168. data/lib/sqa/indicator/head_and_shoulders_pattern.rb +0 -26
  169. data/lib/sqa/indicator/market_profile.rb +0 -32
  170. data/lib/sqa/indicator/mean_reversion.rb +0 -37
  171. data/lib/sqa/indicator/momentum.rb +0 -28
  172. data/lib/sqa/indicator/moving_average_convergence_divergence.rb +0 -29
  173. data/lib/sqa/indicator/peaks_and_valleys.rb +0 -29
  174. data/lib/sqa/indicator/predict_next_value.rb +0 -202
  175. data/lib/sqa/indicator/relative_strength_index.rb +0 -47
  176. data/lib/sqa/indicator/simple_moving_average.rb +0 -24
  177. data/lib/sqa/indicator/simple_moving_average_trend.rb +0 -32
  178. data/lib/sqa/indicator/stochastic_oscillator.rb +0 -68
  179. data/lib/sqa/indicator/true_range.rb +0 -39
  180. data/lib/sqa/trade.rb +0 -26
data/CLAUDE.md ADDED
@@ -0,0 +1,653 @@
1
+ # CLAUDE.md
2
+
3
+ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
4
+
5
+ ## Project Overview
6
+
7
+ SQA (Simple Qualitative Analysis) is a Ruby library for stock market technical analysis designed for educational purposes. It provides high-performance data structures, trading strategies, and integrates with the `sqa-tai` gem for 150+ technical indicators.
8
+
9
+ **Important:** SQA is a **library-only gem** with no CLI functionality. The `sqa-console` executable launches an IRB console for interactive experimentation.
10
+
11
+ ## Common Development Commands
12
+
13
+ ### Running Tests
14
+ ```bash
15
+ rake test # Run unit tests with Minitest
16
+ just test # Alternative: Run tests via justfile
17
+ just coverage # Run tests and open coverage report
18
+ ```
19
+
20
+ ### Building and Installing
21
+ ```bash
22
+ rake install # Install gem locally
23
+ just install # Install with TOC update and man page generation
24
+ rake build # Build gem package
25
+ ```
26
+
27
+ ### Code Quality
28
+ ```bash
29
+ just flay # Static code analysis for duplication
30
+ ```
31
+
32
+ ### Documentation
33
+ ```bash
34
+ rake toc # Update README table of contents
35
+ rake man # Generate man pages
36
+ ```
37
+
38
+ ### Interactive Console
39
+ ```bash
40
+ sqa-console # Launch IRB with SQA library loaded
41
+ ```
42
+
43
+ ## Architecture Overview
44
+
45
+ ### Core Module Structure
46
+ - **SQA::Stock**: Primary domain object representing a stock with price history and metadata
47
+ - **SQA::DataFrame**: High-performance wrapper around Polars for time series data manipulation
48
+ - **SQA::DataFrame::Data**: Stock metadata storage (ticker, name, exchange, source, indicators, overview)
49
+ - **SQAI / SQA::TAI**: Access to 150+ technical indicators from `sqa-tai` gem (TA-Lib wrapper)
50
+ - **SQA::Strategy**: Trading strategy framework with 12+ built-in strategies
51
+ - **SQA::Portfolio**: Position and trade tracking with P&L calculations (265 lines)
52
+ - **SQA::Backtest**: Strategy simulation with comprehensive performance metrics (345 lines)
53
+ - **SQA::Ticker**: Stock symbol validation and lookup
54
+ - **SQA::Config**: Configuration management with YAML/TOML support
55
+
56
+ ### Advanced Module Structure
57
+ - **SQA::StrategyGenerator**: Reverse-engineer profitable trades to discover patterns (690 lines, enhanced with Pattern Context)
58
+ - **SQA::GeneticProgram**: Evolutionary algorithm for strategy parameter optimization (259 lines)
59
+ - **SQA::Strategy::KBS**: RETE-based forward-chaining inference engine (454 lines)
60
+ - **SQA::Stream**: Real-time price stream processor with callbacks (343 lines)
61
+ - **SQA::FPOP**: Future Period Loss/Profit analysis utilities (243 lines)
62
+ - **SQA::MarketRegime**: Bull/bear/sideways market detection with volatility analysis (176 lines)
63
+ - **SQA::SeasonalAnalyzer**: Calendar-dependent pattern discovery (monthly/quarterly) (185 lines)
64
+ - **SQA::SectorAnalyzer**: Cross-stock analysis with KBS blackboards per sector (242 lines)
65
+ - **SQA::RiskManager**: Comprehensive risk management with VaR, CVaR, position sizing (566 lines)
66
+ - **SQA::PortfolioOptimizer**: Multi-objective portfolio optimization and rebalancing (389 lines)
67
+ - **SQA::Ensemble**: Strategy combination with voting and meta-learning (358 lines)
68
+ - **SQA::MultiTimeframe**: Multi-timeframe analysis and trend alignment (398 lines)
69
+ - **SQA::PatternMatcher**: Pattern similarity search with forecasting (567 lines)
70
+
71
+ ### Data Flow
72
+
73
+ **Basic Flow:**
74
+ 1. Create `SQA::Stock` with ticker symbol
75
+ 2. Stock fetches data from Alpha Vantage or Yahoo Finance
76
+ 3. Data stored in Polars-based `SQA::DataFrame`
77
+ 4. Apply technical indicators via `SQAI` / `SQA::TAI` (from sqa-tai gem)
78
+ 5. Execute trading strategies to generate buy/sell/hold signals
79
+ 6. Analyze results with statistical functions
80
+
81
+ **Advanced Workflows:**
82
+
83
+ **Backtesting:**
84
+ 1. Load historical data via `SQA::Stock`
85
+ 2. Create `SQA::Backtest` with stock, strategy, and capital
86
+ 3. Backtest simulates trades using `SQA::Portfolio`
87
+ 4. Returns `Results` with metrics: Sharpe ratio, max drawdown, win rate, etc.
88
+
89
+ **Pattern Discovery:**
90
+ 1. `SQA::StrategyGenerator` scans historical data for profitable points
91
+ 2. Identifies indicator states at each profitable entry
92
+ 3. Mines patterns from indicator combinations
93
+ 4. Generates executable strategies from patterns
94
+ 5. Strategies can be backtested or used live
95
+
96
+ **Parameter Optimization:**
97
+ 1. `SQA::GeneticProgram` defines parameter space (genes)
98
+ 2. Creates random population of strategies
99
+ 3. Evaluates fitness via backtesting
100
+ 4. Evolves through selection, crossover, mutation
101
+ 5. Returns best individual with optimal parameters
102
+
103
+ **Real-Time Trading:**
104
+ 1. `SQA::Stream` receives live price updates
105
+ 2. Maintains rolling window of recent data
106
+ 3. Calculates indicators on-the-fly (with caching)
107
+ 4. Executes multiple strategies in parallel
108
+ 5. Aggregates signals and fires callbacks
109
+ 6. Callbacks can execute trades, send alerts, log data
110
+
111
+ **FPL Analysis:**
112
+ 1. `SQA::FPOP.fpl(prices, fpop: 10)` calculates min/max future deltas
113
+ 2. `SQA::FPOP.fpl_analysis(prices, fpop: 10)` adds risk metrics and direction
114
+ 3. `SQA::FPOP.filter_by_quality()` filters by magnitude, risk, direction
115
+ 4. Integration with `StrategyGenerator` via `max_fpl_risk` parameter
116
+ 5. DataFrame convenience methods: `df.fpl()` and `df.fpl_analysis()`
117
+
118
+ **Market Regime Analysis:**
119
+ 1. `SQA::MarketRegime.detect(stock)` classifies current market
120
+ 2. Returns regime type (bull/bear/sideways), volatility, and strength
121
+ 3. `detect_history(stock)` identifies regime changes over time
122
+ 4. `split_by_regime(stock)` groups data by regime periods
123
+ 5. Enables regime-specific strategy selection
124
+
125
+ **Seasonal Pattern Discovery:**
126
+ 1. `SQA::SeasonalAnalyzer.analyze(stock)` finds calendar patterns
127
+ 2. Identifies best/worst months and quarters
128
+ 3. `detect_seasonality()` determines if patterns are significant
129
+ 4. `filter_by_months()` and `filter_by_quarters()` extract seasonal data
130
+ 5. Enables time-of-year pattern validation
131
+
132
+ **Sector Analysis:**
133
+ 1. `SQA::SectorAnalyzer` creates KBS blackboard per sector
134
+ 2. Stocks registered with sector classification
135
+ 3. `discover_sector_patterns()` finds cross-stock patterns
136
+ 4. `detect_sector_regime()` gets consensus market view
137
+ 5. Persistent SQLite storage for each sector's knowledge base
138
+
139
+ **Context-Aware Pattern Discovery:**
140
+ 1. `StrategyGenerator.discover_context_aware_patterns()` adds metadata
141
+ 2. Patterns tagged with market regime, valid months/quarters, sector
142
+ 3. `PatternContext.valid_for?(date, regime, sector)` runtime validation
143
+ 4. `walk_forward_validate()` prevents overfitting with out-of-sample testing
144
+ 5. Patterns know when they should and shouldn't be used
145
+
146
+ **Risk Management:**
147
+ 1. `SQA::RiskManager.var(returns, confidence: 0.95)` calculates Value at Risk
148
+ 2. `SQA::RiskManager.cvar(returns)` for Conditional VaR (Expected Shortfall)
149
+ 3. Position sizing: `kelly_criterion()`, `fixed_fractional()`, `percent_volatility()`
150
+ 4. Risk metrics: `sharpe_ratio()`, `sortino_ratio()`, `calmar_ratio()`
151
+ 5. `max_drawdown()` and `monte_carlo_simulation()` for risk assessment
152
+
153
+ **Portfolio Optimization:**
154
+ 1. `SQA::PortfolioOptimizer.maximum_sharpe(returns_matrix)` finds optimal weights
155
+ 2. `minimum_variance()` and `risk_parity()` for conservative allocations
156
+ 3. `efficient_frontier()` generates risk/return curve
157
+ 4. `multi_objective()` optimizes multiple goals simultaneously
158
+ 5. `rebalance()` calculates trades needed to reach target allocation
159
+
160
+ **Ensemble Strategies:**
161
+ 1. `SQA::Ensemble.new(strategies: [...], voting_method: :majority)` combines strategies
162
+ 2. Voting methods: `:majority`, `:weighted`, `:unanimous`, `:confidence`
163
+ 3. `update_weight()` adjusts based on performance
164
+ 4. `rotate()` selects best strategy for current market conditions
165
+ 5. `backtest_comparison()` evaluates ensemble vs individuals
166
+
167
+ **Multi-Timeframe Analysis:**
168
+ 1. `SQA::MultiTimeframe.new(stock: stock)` converts daily → weekly → monthly
169
+ 2. `trend_alignment()` checks if all timeframes agree
170
+ 3. `signal()` combines higher timeframe trend with lower timeframe timing
171
+ 4. `support_resistance()` finds levels that appear across multiple timeframes
172
+ 5. `detect_divergence()` identifies price/indicator disagreement
173
+
174
+ **Pattern Similarity Search:**
175
+ 1. `SQA::PatternMatcher.find_similar(lookback: 10)` finds historical matches
176
+ 2. Methods: `:euclidean`, `:dtw` (Dynamic Time Warping), `:correlation`
177
+ 3. `forecast()` predicts future moves based on similar past patterns
178
+ 4. `detect_chart_pattern()` finds double tops/bottoms, head & shoulders, triangles
179
+ 5. `cluster_patterns()` groups similar patterns with k-means
180
+
181
+ ### Key Design Patterns
182
+ - **Plugin Architecture**: Strategies are pluggable modules
183
+ - **Data Source Abstraction**: Multiple data providers (Alpha Vantage, Yahoo Finance) with common interface
184
+ - **Delegation Pattern**: DataFrame delegates to Polars for high-performance operations
185
+ - **Configuration Hierarchy**: defaults < environment variables < config file
186
+
187
+ ## Important Implementation Notes
188
+
189
+ ### Data Sources
190
+ - **Alpha Vantage API** requires `AV_API_KEY` or `ALPHAVANTAGE_API_KEY` environment variable
191
+ - **Yahoo Finance** scraping available as fallback (no API, less reliable)
192
+ - CSV file imports supported for historical data (place in data directory as `ticker.csv`)
193
+
194
+ ### Configuration
195
+ - Config files: YAML or TOML in `~/.sqa.*`
196
+ - Data directory: `~/sqa_data/` (default, configurable)
197
+ - Environment variables:
198
+ - `AV_API_KEY` or `ALPHAVANTAGE_API_KEY` - Alpha Vantage API key
199
+ - Custom config via `SQA::Config.new(data_dir: '...')`
200
+
201
+ ### DataFrame Implementation
202
+ - Uses `polars-df` gem (Rust-backed, blazingly fast)
203
+ - Wraps Polars::DataFrame with convenience methods
204
+ - Custom statistics via `lite-statistics` gem monkey patches on Arrays
205
+ - **Always** prefer column-based operations over row iterations for performance
206
+ - Access underlying Polars DataFrame via `.data` attribute
207
+
208
+ ### Technical Indicators
209
+ - **All indicators** provided by separate `sqa-tai` gem
210
+ - `sqa-tai` wraps TA-Lib C library (industry standard)
211
+ - Access via `SQAI.indicator_name(prices, options)` or `SQA::TAI.indicator_name(...)`
212
+ - 150+ indicators available: SMA, EMA, RSI, MACD, Bollinger Bands, ADX, ATR, etc.
213
+ - See: https://github.com/MadBomber/sqa-tai
214
+
215
+ ### Testing Approach
216
+ - Minitest framework in `/test/` directory
217
+ - SimpleCov for coverage reporting
218
+ - Test data fixtures in `test/test_helper.rb`
219
+ - **Note:** Indicator tests may need updates after migrating to sqa-tai (different return values)
220
+
221
+ ## Development Guidelines
222
+
223
+ ### When Adding New Strategies
224
+ 1. Create new file in `lib/sqa/strategy/`
225
+ 2. Define class under `SQA::Strategy::` namespace
226
+ 3. Implement `self.trade(vector)` class method
227
+ 4. Return `:buy`, `:sell`, or `:hold`
228
+ 5. Add corresponding test in `test/strategy/`
229
+
230
+ **Example:**
231
+ ```ruby
232
+ class SQA::Strategy::MyStrategy
233
+ def self.trade(vector)
234
+ # vector is an OpenStruct with indicator values
235
+ if vector.rsi < 30
236
+ :buy
237
+ elsif vector.rsi > 70
238
+ :sell
239
+ else
240
+ :hold
241
+ end
242
+ end
243
+ end
244
+ ```
245
+
246
+ ### Working with DataFrames
247
+ - Use Polars native operations when possible (accessed via `df.data`)
248
+ - Avoid Ruby loops over rows for performance
249
+ - Leverage vectorized operations
250
+ - Example: `df.data["close_price"]` returns Polars::Series
251
+
252
+ ### Working with Stock Metadata (DataFrame::Data)
253
+ - **Purpose**: Stores stock metadata separate from price/volume DataFrame
254
+ - **Attributes**: ticker, name, exchange, source, indicators, overview
255
+ - **Dual initialization**:
256
+ - From hash: `SQA::DataFrame::Data.new(JSON.parse(json_string))`
257
+ - From keywords: `SQA::DataFrame::Data.new(ticker: 'AAPL', source: :alpha_vantage, indicators: {})`
258
+ - **JSON serialization**: `data.to_json` for persistence
259
+ - **Used by**: `SQA::Stock` to persist metadata in `~/sqa_data/ticker.json`
260
+ - **File location**: `lib/sqa/data_frame/data.rb`
261
+
262
+ ### Working with Indicators
263
+ - **Do NOT add indicators to SQA** - contribute to `sqa-tai` gem instead
264
+ - Use indicators: `SQAI.sma(prices_array, period: 20)`
265
+ - All indicators work on Ruby Arrays, not DataFrames
266
+ - Extract price array first: `prices = stock.df["close_price"].to_a`
267
+
268
+ ### API Integration
269
+ - New data sources go in `lib/sqa/data_frame/`
270
+ - Follow existing adapter pattern (see `alpha_vantage.rb` and `yahoo_finance.rb`)
271
+ - Handle rate limiting and errors gracefully
272
+ - Return Polars-compatible data structures
273
+
274
+ ### Adding New Data Sources
275
+ 1. Create `lib/sqa/data_frame/my_source.rb`
276
+ 2. Define class `SQA::DataFrame::MySource`
277
+ 3. Implement `self.recent(ticker, **options)` method
278
+ 4. Return data in format compatible with Polars::DataFrame
279
+ 5. Add mapping for column names if needed
280
+
281
+ ## Critical Constraints
282
+
283
+ - This is an **EDUCATIONAL tool** - maintain clear disclaimers
284
+ - Do NOT remove financial risk warnings from README or docs
285
+ - **No CLI** - this is a library, not a command-line application
286
+ - Performance matters - prefer vectorized DataFrame operations
287
+ - Maintain backward compatibility with existing data file formats
288
+ - API keys from environment variables only (no api_key_manager)
289
+
290
+ ### Testing Guidelines
291
+ - All new features should have corresponding tests in `/test/`
292
+ - Integration tests require `RUN_INTEGRATION_TESTS=1` environment variable
293
+ - Use `skip` for tests requiring network access or long runtimes
294
+ - Portfolio, Backtest, Stream have comprehensive test coverage
295
+ - GP and StrategyGenerator tests available but skipped by default (long running)
296
+
297
+ ## File Structure
298
+
299
+ ```
300
+ lib/
301
+ ├── api/
302
+ │ └── alpha_vantage_api.rb # Alpha Vantage API client (462 lines)
303
+ ├── patches/
304
+ │ └── string.rb # String helpers (camelize, constantize, underscore)
305
+ └── sqa/
306
+ ├── backtest.rb # ✨ NEW: Backtesting framework (345 lines)
307
+ ├── config.rb # Configuration management
308
+ ├── data_frame.rb # Polars DataFrame wrapper
309
+ ├── data_frame/
310
+ │ ├── alpha_vantage.rb # Alpha Vantage data adapter
311
+ │ ├── data.rb # Stock metadata storage class (93 lines)
312
+ │ └── yahoo_finance.rb # Yahoo Finance scraper
313
+ ├── errors.rb # Error classes
314
+ ├── ensemble.rb # ✨ NEW: Strategy combination and voting (358 lines)
315
+ ├── fpop.rb # ✨ NEW: Future Period Loss/Profit analysis (243 lines)
316
+ ├── gp.rb # ✨ NEW: Genetic programming (259 lines, COMPLETE)
317
+ ├── indicator.rb # Delegates to sqa-tai gem
318
+ ├── init.rb # Module initialization
319
+ ├── market_regime.rb # ✨ NEW: Market regime detection (176 lines)
320
+ ├── multi_timeframe.rb # ✨ NEW: Multi-timeframe analysis (398 lines)
321
+ ├── pattern_matcher.rb # ✨ NEW: Pattern similarity search (567 lines)
322
+ ├── portfolio.rb # ✨ NEW: Portfolio management (265 lines, COMPLETE)
323
+ ├── portfolio_optimizer.rb # ✨ NEW: Portfolio optimization (389 lines)
324
+ ├── risk_manager.rb # ✨ NEW: Risk management (566 lines)
325
+ ├── seasonal_analyzer.rb # ✨ NEW: Seasonal pattern discovery (185 lines)
326
+ ├── sector_analyzer.rb # ✨ NEW: Sector analysis with KBS (242 lines)
327
+ ├── stock.rb # Stock class with data management
328
+ ├── strategy.rb # Strategy framework
329
+ ├── strategy/
330
+ │ ├── bollinger_bands.rb # ✨ NEW: Bollinger Bands strategy
331
+ │ ├── common.rb # Shared strategy utilities
332
+ │ ├── consensus.rb # Consensus from multiple strategies
333
+ │ ├── ema.rb # EMA-based strategy
334
+ │ ├── kbs_strategy.rb # ✨ NEW: RETE-based KBS (454 lines)
335
+ │ ├── macd.rb # ✨ NEW: MACD crossover strategy
336
+ │ ├── mp.rb # Market Profile strategy
337
+ │ ├── mr.rb # Mean Reversion strategy
338
+ │ ├── random.rb # Random signals (testing)
339
+ │ ├── rsi.rb # RSI-based strategy
340
+ │ ├── sma.rb # SMA-based strategy
341
+ │ ├── stochastic.rb # ✨ NEW: Stochastic oscillator strategy
342
+ │ └── volume_breakout.rb # ✨ NEW: Volume breakout strategy
343
+ ├── stream.rb # ✨ NEW: Real-time price streaming (343 lines)
344
+ ├── strategy_generator.rb # ✨ NEW: Pattern discovery (690 lines)
345
+ ├── ticker.rb # Ticker validation
346
+ └── version.rb # Version constant
347
+
348
+ examples/
349
+ ├── README.md # ✨ NEW: Comprehensive examples guide
350
+ ├── advanced_features_example.rb # ✨ NEW: All advanced features demo (396 lines)
351
+ ├── fpop_analysis_example.rb # ✨ NEW: FPL analysis utilities (191 lines)
352
+ ├── genetic_programming_example.rb # ✨ NEW: GP parameter evolution
353
+ ├── kbs_strategy_example.rb # ✨ NEW: RETE rule-based trading
354
+ ├── pattern_context_example.rb # ✨ NEW: Context-aware patterns (280 lines)
355
+ ├── realtime_stream_example.rb # ✨ NEW: Live price processing
356
+ └── strategy_generator_example.rb # ✨ NEW: Pattern mining
357
+
358
+ test/
359
+ ├── backtest_test.rb # ✨ NEW: Backtest tests
360
+ ├── ensemble_test.rb # ✨ NEW: Ensemble methods tests (109 lines, 12 tests)
361
+ ├── fpop_test.rb # ✨ NEW: FPL analysis tests (154 lines)
362
+ ├── gp_test.rb # ✨ NEW: Genetic programming tests
363
+ ├── market_regime_test.rb # ✨ NEW: Market regime tests (165 lines)
364
+ ├── multi_timeframe_test.rb # ✨ NEW: Multi-timeframe tests (77 lines, 7 tests)
365
+ ├── pattern_context_test.rb # ✨ NEW: Pattern context tests (177 lines)
366
+ ├── pattern_context_integration_test.rb # ✨ NEW: Integration tests (342 lines)
367
+ ├── pattern_matcher_test.rb # ✨ NEW: Pattern matching tests (155 lines, 15 tests)
368
+ ├── portfolio_test.rb # ✨ NEW: Portfolio tests
369
+ ├── portfolio_optimizer_test.rb # ✨ NEW: Portfolio optimization tests (122 lines, 10 tests)
370
+ ├── risk_manager_test.rb # ✨ NEW: Risk management tests (183 lines, 22 tests)
371
+ ├── seasonal_analyzer_test.rb # ✨ NEW: Seasonal analysis tests (203 lines)
372
+ ├── sector_analyzer_test.rb # ✨ NEW: Sector analysis tests (162 lines)
373
+ ├── stream_test.rb # ✨ NEW: Stream processor tests
374
+ ├── strategy_generator_test.rb # ✨ NEW: Strategy generator tests
375
+ ├── data_frame_test.rb # DataFrame tests
376
+ ├── test_helper.rb # Test configuration
377
+ └── indicator/ # Indicator tests (legacy, may need updates)
378
+ ```
379
+
380
+ **Line Counts:**
381
+ - Core: ~2,000 lines (DataFrame, Stock, Strategy framework)
382
+ - Advanced Features: ~5,428 lines (13 advanced modules)
383
+ * Pattern Context System: 1,089 lines (FPOP, MarketRegime, SeasonalAnalyzer, SectorAnalyzer)
384
+ * Trading Infrastructure: 867 lines (Portfolio, Backtest, Stream)
385
+ * Intelligence: 1,403 lines (GP, KBS, StrategyGenerator)
386
+ * New Advanced Features: 2,278 lines (RiskManager, PortfolioOptimizer, Ensemble, MultiTimeframe, PatternMatcher)
387
+ - Tests: ~2,650 lines (17 test files, 132+ tests)
388
+ - Examples: ~2,250 lines (8 comprehensive examples)
389
+ - Total: ~12,328 lines
390
+
391
+ ## Common Gotchas
392
+
393
+ 1. **DataFrame vs Polars**: `df` is SQA::DataFrame, `df.data` is Polars::DataFrame
394
+ 2. **Indicators need Arrays**: Extract data with `.to_a` before passing to indicators
395
+ 3. **No CLI commands**: Previous CLI functionality has been removed
396
+ 4. **Indicators in separate gem**: Technical indicators are in `sqa-tai`, not SQA
397
+ 5. **API key format changed**: Use `AV_API_KEY` not `AV_API_KEYS` (singular)
398
+ 6. **Strategies need OpenStruct**: Pass data to strategies as OpenStruct with named fields
399
+
400
+ ## Advanced Features Quick Reference
401
+
402
+ ### Portfolio Management
403
+ ```ruby
404
+ portfolio = SQA::Portfolio.new(initial_cash: 10_000, commission: 1.0)
405
+ portfolio.buy('AAPL', shares: 10, price: 150.0)
406
+ portfolio.sell('AAPL', shares: 5, price: 160.0)
407
+ portfolio.value({ 'AAPL' => 165.0 })
408
+ ```
409
+
410
+ ### Backtesting
411
+ ```ruby
412
+ backtest = SQA::Backtest.new(stock: stock, strategy: SQA::Strategy::RSI)
413
+ results = backtest.run
414
+ puts "Return: #{results.total_return}%, Sharpe: #{results.sharpe_ratio}"
415
+ ```
416
+
417
+ ### Strategy Generator
418
+ ```ruby
419
+ gen = SQA::StrategyGenerator.new(stock: stock, min_gain_percent: 10.0)
420
+ patterns = gen.discover_patterns
421
+ strategy = gen.generate_strategy(pattern_index: 0)
422
+ ```
423
+
424
+ ### Genetic Programming
425
+ ```ruby
426
+ gp = SQA::GeneticProgram.new(stock: stock, population_size: 50)
427
+ gp.define_genes(rsi_period: (7..30).to_a)
428
+ gp.fitness { |genes| backtest_with(genes).total_return }
429
+ best = gp.evolve
430
+ ```
431
+
432
+ ### Knowledge-Based Strategy
433
+ ```ruby
434
+ strategy = SQA::Strategy::KBS.new(load_defaults: false)
435
+ strategy.add_rule :my_rule do
436
+ on :rsi, { level: :oversold }
437
+ on :macd, { crossover: :bullish }
438
+ perform { assert(:signal, { action: :buy }) }
439
+ end
440
+ ```
441
+
442
+ ### Real-Time Streaming
443
+ ```ruby
444
+ stream = SQA::Stream.new(ticker: 'AAPL', strategies: [SQA::Strategy::RSI])
445
+ stream.on_signal { |signal, data| execute_trade(signal, data) }
446
+ stream.update(price: 150.25, volume: 1_000_000)
447
+ ```
448
+
449
+ ### FPL Analysis
450
+ ```ruby
451
+ # Basic FPL calculation
452
+ prices = stock.df["adj_close_price"].to_a
453
+ fpl_data = SQA::FPOP.fpl(prices, fpop: 10)
454
+ # => [[min_delta, max_delta], ...]
455
+
456
+ # Comprehensive analysis
457
+ analysis = SQA::FPOP.fpl_analysis(prices, fpop: 10)
458
+ puts "Risk: #{analysis[:risk]}%, Direction: #{analysis[:direction]}"
459
+
460
+ # Filter high-quality opportunities
461
+ filtered = SQA::FPOP.filter_by_quality(
462
+ analysis,
463
+ min_magnitude: 5.0,
464
+ max_risk: 25.0,
465
+ directions: [:UP]
466
+ )
467
+ ```
468
+
469
+ ### Market Regime Detection
470
+ ```ruby
471
+ # Detect current regime
472
+ regime = SQA::MarketRegime.detect(stock)
473
+ puts "Regime: #{regime[:type]}" # => :bull, :bear, or :sideways
474
+ puts "Volatility: #{regime[:volatility]}" # => :low, :medium, :high
475
+
476
+ # Get regime history
477
+ history = SQA::MarketRegime.detect_history(stock, window: 60)
478
+
479
+ # Split data by regime
480
+ splits = SQA::MarketRegime.split_by_regime(stock)
481
+ bull_periods = splits[:bull]
482
+ ```
483
+
484
+ ### Seasonal Analysis
485
+ ```ruby
486
+ # Analyze seasonal patterns
487
+ seasonal = SQA::SeasonalAnalyzer.analyze(stock)
488
+ puts "Best months: #{seasonal[:best_months]}" # => [10, 11, 12]
489
+ puts "Best quarters: #{seasonal[:best_quarters]}" # => [4, 1]
490
+
491
+ # Filter for Q4 only
492
+ q4_data = SQA::SeasonalAnalyzer.filter_by_quarters(stock, [4])
493
+ ```
494
+
495
+ ### Sector Analysis
496
+ ```ruby
497
+ analyzer = SQA::SectorAnalyzer.new
498
+ analyzer.add_stock('AAPL', sector: :technology)
499
+ analyzer.add_stock('MSFT', sector: :technology)
500
+
501
+ # Detect sector regime
502
+ tech_stocks = ['AAPL', 'MSFT'].map { |t| SQA::Stock.new(ticker: t) }
503
+ regime = analyzer.detect_sector_regime(:technology, tech_stocks)
504
+
505
+ # Find sector-wide patterns
506
+ patterns = analyzer.discover_sector_patterns(:technology, tech_stocks)
507
+ ```
508
+
509
+ ### Context-Aware Pattern Discovery
510
+ ```ruby
511
+ # Discover patterns with context
512
+ generator = SQA::StrategyGenerator.new(stock: stock, fpop: 10)
513
+ patterns = generator.discover_context_aware_patterns(
514
+ analyze_regime: true,
515
+ analyze_seasonal: true,
516
+ sector: :technology
517
+ )
518
+
519
+ # Walk-forward validation
520
+ validated = generator.walk_forward_validate(
521
+ train_size: 250,
522
+ test_size: 60
523
+ )
524
+
525
+ # Runtime validation
526
+ pattern = patterns.first
527
+ valid = pattern.context.valid_for?(
528
+ date: Date.today,
529
+ regime: :bull,
530
+ sector: :technology
531
+ )
532
+ ```
533
+
534
+ ### Risk Management
535
+ ```ruby
536
+ # Value at Risk
537
+ returns = prices.each_cons(2).map { |a, b| (b - a) / a }
538
+ var_95 = SQA::RiskManager.var(returns, confidence: 0.95, method: :historical)
539
+ cvar_95 = SQA::RiskManager.cvar(returns, confidence: 0.95)
540
+
541
+ # Position sizing
542
+ position = SQA::RiskManager.kelly_criterion(
543
+ win_rate: 0.60,
544
+ avg_win: 0.10,
545
+ avg_loss: 0.05,
546
+ capital: 10_000
547
+ )
548
+
549
+ # Risk metrics
550
+ sharpe = SQA::RiskManager.sharpe_ratio(returns)
551
+ sortino = SQA::RiskManager.sortino_ratio(returns)
552
+ dd = SQA::RiskManager.max_drawdown(prices)
553
+ ```
554
+
555
+ ### Portfolio Optimization
556
+ ```ruby
557
+ # Multiple stocks returns
558
+ returns_matrix = stocks.map do |stock|
559
+ stock.df["adj_close_price"].to_a.each_cons(2).map { |a, b| (b - a) / a }
560
+ end
561
+
562
+ # Maximum Sharpe
563
+ max_sharpe = SQA::PortfolioOptimizer.maximum_sharpe(returns_matrix)
564
+ # => { weights: [0.4, 0.3, 0.3], sharpe: 1.5, return: 0.12, volatility: 0.15 }
565
+
566
+ # Minimum variance
567
+ min_var = SQA::PortfolioOptimizer.minimum_variance(returns_matrix)
568
+
569
+ # Multi-objective
570
+ multi = SQA::PortfolioOptimizer.multi_objective(
571
+ returns_matrix,
572
+ objectives: { maximize_return: 0.4, minimize_volatility: 0.3, minimize_drawdown: 0.3 }
573
+ )
574
+ ```
575
+
576
+ ### Ensemble Strategies
577
+ ```ruby
578
+ # Create ensemble
579
+ ensemble = SQA::Ensemble.new(
580
+ strategies: [SQA::Strategy::RSI, SQA::Strategy::MACD],
581
+ voting_method: :majority
582
+ )
583
+
584
+ # Get ensemble signal
585
+ signal = ensemble.signal(vector) # => :buy, :sell, or :hold
586
+
587
+ # Rotate based on market
588
+ selected = ensemble.rotate(stock)
589
+ ```
590
+
591
+ ### Multi-Timeframe Analysis
592
+ ```ruby
593
+ # Create analyzer
594
+ mta = SQA::MultiTimeframe.new(stock: stock)
595
+
596
+ # Check trend alignment
597
+ alignment = mta.trend_alignment
598
+ # => { daily: :up, weekly: :up, monthly: :up, aligned: true, direction: :bullish }
599
+
600
+ # Multi-timeframe signal
601
+ signal = mta.signal(
602
+ strategy_class: SQA::Strategy::RSI,
603
+ higher_timeframe: :weekly,
604
+ lower_timeframe: :daily
605
+ )
606
+
607
+ # Find strong support/resistance
608
+ levels = mta.support_resistance(tolerance: 0.02)
609
+ ```
610
+
611
+ ### Pattern Similarity Search
612
+ ```ruby
613
+ # Create matcher
614
+ matcher = SQA::PatternMatcher.new(stock: stock)
615
+
616
+ # Find similar patterns
617
+ similar = matcher.find_similar(lookback: 10, num_matches: 5, method: :euclidean)
618
+ # => [{ distance: 0.05, future_return: 0.12, pattern: [...] }, ...]
619
+
620
+ # Forecast based on similar patterns
621
+ forecast = matcher.forecast(lookback: 10, forecast_periods: 5)
622
+ # => { forecast_price: 155.0, forecast_return: 0.03, confidence_interval_95: [150, 160] }
623
+
624
+ # Detect chart patterns
625
+ patterns = matcher.detect_chart_pattern(:double_top)
626
+ ```
627
+
628
+ ## Quick Reference
629
+
630
+ ```ruby
631
+ require 'sqa'
632
+
633
+ # Initialize
634
+ SQA.init
635
+
636
+ # Load stock data
637
+ stock = SQA::Stock.new(ticker: 'AAPL')
638
+
639
+ # Get price array
640
+ prices = stock.df["adj_close_price"].to_a
641
+
642
+ # Calculate indicators
643
+ sma = SQAI.sma(prices, period: 20)
644
+ rsi = SQAI.rsi(prices, period: 14)
645
+
646
+ # Execute strategies
647
+ require 'ostruct'
648
+ vector = OpenStruct.new(rsi: rsi.last, prices: prices)
649
+
650
+ strategy = SQA::Strategy.new
651
+ strategy.add SQA::Strategy::RSI
652
+ signals = strategy.execute(vector) # => [:buy, :sell, :hold]
653
+ ```