sqa 0.0.24 → 0.0.31

data/lib/sqa/backtest.rb

@@ -0,0 +1,329 @@
+# lib/sqa/backtest.rb
+# frozen_string_literal: true
+
+require 'date'
+require_relative 'portfolio'
+
+class SQA::Backtest
+  attr_reader :stock, :strategy, :portfolio, :results, :equity_curve
+
+  # Represents the results of a backtest
+  class Results
+    attr_accessor :total_return, :annualized_return, :sharpe_ratio, :max_drawdown,
+                  :total_trades, :winning_trades, :losing_trades, :win_rate,
+                  :average_win, :average_loss, :profit_factor,
+                  :start_date, :end_date, :initial_capital, :final_value
+
+    def initialize
+      @total_return = 0.0
+      @annualized_return = 0.0
+      @sharpe_ratio = 0.0
+      @max_drawdown = 0.0
+      @total_trades = 0
+      @winning_trades = 0
+      @losing_trades = 0
+      @win_rate = 0.0
+      @average_win = 0.0
+      @average_loss = 0.0
+      @profit_factor = 0.0
+    end
+
+    def to_h
+      {
+        total_return: total_return,
+        annualized_return: annualized_return,
+        sharpe_ratio: sharpe_ratio,
+        max_drawdown: max_drawdown,
+        total_trades: total_trades,
+        winning_trades: winning_trades,
+        losing_trades: losing_trades,
+        win_rate: win_rate,
+        average_win: average_win,
+        average_loss: average_loss,
+        profit_factor: profit_factor,
+        start_date: start_date,
+        end_date: end_date,
+        initial_capital: initial_capital,
+        final_value: final_value
+      }
+    end
+
+    def summary
+      <<~SUMMARY
+        Backtest Results
+        ================
+        Period: #{start_date} to #{end_date}
+        Initial Capital: $#{initial_capital.round(2)}
+        Final Value: $#{final_value.round(2)}
+
+        Performance Metrics:
+        - Total Return: #{(total_return * 100).round(2)}%
+        - Annualized Return: #{(annualized_return * 100).round(2)}%
+        - Sharpe Ratio: #{sharpe_ratio.round(2)}
+        - Maximum Drawdown: #{(max_drawdown * 100).round(2)}%
+
+        Trade Statistics:
+        - Total Trades: #{total_trades}
+        - Winning Trades: #{winning_trades}
+        - Losing Trades: #{losing_trades}
+        - Win Rate: #{(win_rate * 100).round(2)}%
+        - Average Win: $#{average_win.round(2)}
+        - Average Loss: $#{average_loss.round(2)}
+        - Profit Factor: #{profit_factor.round(2)}
+      SUMMARY
+    end
+  end
+
+  # Initialize a backtest
+  # @param stock [SQA::Stock] Stock to backtest
+  # @param strategy [SQA::Strategy, Proc] Strategy or callable that returns :buy, :sell, or :hold
+  # @param start_date [Date, String] Start date for backtest
+  # @param end_date [Date, String] End date for backtest
+  # @param initial_capital [Float] Starting capital
+  # @param commission [Float] Commission per trade
+  # @param position_size [Symbol, Float] :all_cash or fraction of portfolio per trade
+  def initialize(stock:, strategy:, start_date: nil, end_date: nil,
+                 initial_capital: 10_000.0, commission: 0.0, position_size: :all_cash)
+    @stock = stock
+    @strategy = strategy
+    @start_date = start_date ? Date.parse(start_date.to_s) : Date.parse(stock.df["timestamp"].first)
+    @end_date = end_date ? Date.parse(end_date.to_s) : Date.parse(stock.df["timestamp"].last)
+    @initial_capital = initial_capital
+    @commission = commission
+    @position_size = position_size
+
+    @portfolio = SQA::Portfolio.new(initial_cash: initial_capital, commission: commission)
+    @equity_curve = []  # Track portfolio value over time
+    @results = Results.new
+  end
+
+  # Run the backtest
+  # @return [Results] Backtest results
+  def run
+    # Get data for the backtest period
+    df = @stock.df.data
+
+    # Filter to backtest period
+    timestamps = df["timestamp"].to_a
+    start_idx = timestamps.index { |t| Date.parse(t) >= @start_date } || 0
+    end_idx = timestamps.rindex { |t| Date.parse(t) <= @end_date } || timestamps.length - 1
+
+    prices = df["adj_close_price"].to_a
+    ticker = @stock.ticker.upcase
+
+    # Track current position
+    current_position = nil  # :long, :short, or nil
+
+    # Run through each day
+    (start_idx..end_idx).each do |i|
+      date = Date.parse(timestamps[i])
+      price = prices[i]
+
+      # Get historical prices up to this point for strategy
+      historical_prices = prices[0..i]
+
+      # Generate signal from strategy
+      signal = generate_signal(historical_prices)
+
+      # Execute trades based on signal
+      case signal
+      when :buy
+        if current_position.nil? && can_buy?(price)
+          shares = calculate_shares_to_buy(price)
+          @portfolio.buy(ticker, shares: shares, price: price, date: date)
+          current_position = :long
+        end
+
+      when :sell
+        if current_position == :long
+          pos = @portfolio.position(ticker)
+          @portfolio.sell(ticker, shares: pos.shares, price: price, date: date) if pos
+          current_position = nil
+        end
+      end
+
+      # Record equity curve
+      current_value = @portfolio.value(ticker => price)
+      @equity_curve << { date: date, value: current_value, price: price }
+    end
+
+    # Close any open positions at end
+    if current_position == :long
+      final_price = prices[end_idx]
+      final_date = Date.parse(timestamps[end_idx])
+      pos = @portfolio.position(ticker)
+      @portfolio.sell(ticker, shares: pos.shares, price: final_price, date: final_date) if pos
+    end
+
+    # Calculate results
+    calculate_results
+
+    @results
+  end
+
+  private
+
+  # Generate trading signal from strategy
+  # @param historical_prices [Array<Float>] Price history
+  # @return [Symbol] :buy, :sell, or :hold
+  def generate_signal(historical_prices)
+    if @strategy.respond_to?(:execute)
+      # SQA::Strategy object
+      require 'ostruct'
+
+      # Calculate indicators for strategy
+      vector = OpenStruct.new
+      vector.prices = historical_prices
+
+      # Add common indicators if we have enough data
+      if historical_prices.length >= 14
+        vector.rsi = SQAI.rsi(historical_prices, period: 14).last rescue nil
+      end
+
+      if historical_prices.length >= 20
+        vector.sma_20 = SQAI.sma(historical_prices, period: 20).last rescue nil
+      end
+
+      if historical_prices.length >= 50
+        vector.sma_50 = SQAI.sma(historical_prices, period: 50).last rescue nil
+      end
+
+      signals = @strategy.execute(vector)
+      signals.first || :hold
+    elsif @strategy.respond_to?(:call)
+      # Proc or lambda
+      @strategy.call(historical_prices)
+    elsif @strategy.respond_to?(:trade)
+      # Strategy class
+      require 'ostruct'
+      vector = OpenStruct.new(prices: historical_prices)
+
+      if historical_prices.length >= 14
+        vector.rsi = SQAI.rsi(historical_prices, period: 14).last rescue nil
+      end
+
+      @strategy.trade(vector)
+    else
+      :hold
+    end
+  end
+
+  # Check if we can afford to buy at given price
+  # @param price [Float] Stock price
+  # @return [Boolean] True if we can buy
+  def can_buy?(price)
+    shares = calculate_shares_to_buy(price)
+    shares > 0 && (shares * price + @commission) <= @portfolio.cash
+  end
+
+  # Calculate how many shares to buy
+  # @param price [Float] Stock price
+  # @return [Integer] Number of shares
+  def calculate_shares_to_buy(price)
+    if @position_size == :all_cash
+      # Use all available cash
+      max_shares = (@portfolio.cash - @commission) / price
+      max_shares.floor
+    else
+      # Use fraction of portfolio
+      capital_to_use = @portfolio.cash * @position_size
+      max_shares = (capital_to_use - @commission) / price
+      max_shares.floor
+    end
+  end
+
+  # Calculate backtest results and metrics
+  def calculate_results
+    @results.initial_capital = @initial_capital
+    @results.final_value = @equity_curve.last[:value]
+    @results.start_date = @start_date
+    @results.end_date = @end_date
+
+    # Total return
+    @results.total_return = (@results.final_value - @initial_capital) / @initial_capital
+
+    # Annualized return
+    days = (@end_date - @start_date).to_i
+    years = days / 365.0
+    if years > 0
+      @results.annualized_return = ((1 + @results.total_return) ** (1.0 / years)) - 1
+    end
+
+    # Sharpe ratio (simplified - assumes risk-free rate of 0)
+    returns = calculate_daily_returns
+    if returns.any? && returns.map { |r| r ** 2 }.sum > 0
+      avg_return = returns.sum / returns.size
+      std_dev = Math.sqrt(returns.map { |r| (r - avg_return) ** 2 }.sum / returns.size)
+      @results.sharpe_ratio = std_dev > 0 ? (avg_return / std_dev) * Math.sqrt(252) : 0.0
+    end
+
+    # Maximum drawdown
+    @results.max_drawdown = calculate_max_drawdown
+
+    # Trade statistics
+    calculate_trade_statistics
+  end
+
+  # Calculate daily returns from equity curve
+  # @return [Array<Float>] Array of daily returns
+  def calculate_daily_returns
+    returns = []
+    @equity_curve.each_cons(2) do |prev, curr|
+      returns << (curr[:value] - prev[:value]) / prev[:value]
+    end
+    returns
+  end
+
+  # Calculate maximum drawdown
+  # @return [Float] Maximum drawdown as decimal
+  def calculate_max_drawdown
+    peak = @equity_curve.first[:value]
+    max_dd = 0.0
+
+    @equity_curve.each do |point|
+      if point[:value] > peak
+        peak = point[:value]
+      else
+        dd = (peak - point[:value]) / peak
+        max_dd = dd if dd > max_dd
+      end
+    end
+
+    max_dd
+  end
+
+  # Calculate trade statistics
+  def calculate_trade_statistics
+    trades = @portfolio.trade_history
+    @results.total_trades = trades.count { |t| t.action == :sell }
+
+    # Match buys with sells to calculate P&L per trade
+    trade_pls = []
+    buy_trades = trades.select { |t| t.action == :buy }
+    sell_trades = trades.select { |t| t.action == :sell }
+
+    sell_trades.each_with_index do |sell, i|
+      if i < buy_trades.size
+        buy = buy_trades[i]
+        pl = (sell.price - buy.price) * sell.shares - sell.commission - buy.commission
+        trade_pls << pl
+      end
+    end
+
+    if trade_pls.any?
+      winning = trade_pls.select { |pl| pl > 0 }
+      losing = trade_pls.select { |pl| pl < 0 }
+
+      @results.winning_trades = winning.size
+      @results.losing_trades = losing.size
+      @results.win_rate = winning.size.to_f / trade_pls.size
+      @results.average_win = winning.any? ? winning.sum / winning.size : 0.0
+      @results.average_loss = losing.any? ? losing.sum / losing.size : 0.0
+
+      # Profit factor
+      total_wins = winning.sum
+      total_losses = losing.sum.abs
+      @results.profit_factor = total_losses > 0 ? total_wins / total_losses : 0.0
+    end
+  end
+end

data/lib/sqa/data_frame/alpha_vantage.rb

@@ -3,58 +3,42 @@
 #
 # Using the Alpha Vantage JSON interface
 #
-
+require 'polars'
 
 class SQA::DataFrame
   class AlphaVantage
     CONNECTION  = Faraday.new(url: 'https://www.alphavantage.co')
     HEADERS     = YahooFinance::HEADERS
 
-    # The Alpha Vantage headers are being remapped so that
-    # they match those of the Yahoo Finance CSV file.
-    #
+    # The Alpha Vantage CSV format uses these exact column names:
+    # timestamp, open, high, low, close, volume
+    # We remap them to match Yahoo Finance column names for consistency
     HEADER_MAPPING = {
-      "date"            => HEADERS[0],
-      "open"            => HEADERS[1],
-      "high"            => HEADERS[2],
-      "low"             => HEADERS[3],
-      "close"           => HEADERS[4],
-      "adjusted_close"  => HEADERS[5],
-      "volume"          => HEADERS[6]
+      "timestamp" => HEADERS[0],  # :timestamp (already matches, but explicit)
+      "open"      => HEADERS[1],  # :open_price
+      "high"      => HEADERS[2],  # :high_price
+      "low"       => HEADERS[3],  # :low_price
+      "close"     => HEADERS[4],  # :close_price (AND :adj_close_price - AV doesn't split these)
+      "volume"    => HEADERS[6]   # :volume
     }
 
+    # Transformers applied AFTER column renaming
+    # Alpha Vantage CSV doesn't have adjusted_close, so we only transform what exists
     TRANSFORMERS  = {
-      HEADERS[1] => -> (v) { v.to_f.round(3) },
-      HEADERS[2] => -> (v) { v.to_f.round(3) },
-      HEADERS[3] => -> (v) { v.to_f.round(3) },
-      HEADERS[4] => -> (v) { v.to_f.round(3) },
-      HEADERS[5] => -> (v) { v.to_f.round(3) },
-      HEADERS[6] => -> (v) { v.to_i }
+      HEADERS[1] => -> (v) { v.to_f.round(3) },  # :open_price
+      HEADERS[2] => -> (v) { v.to_f.round(3) },  # :high_price
+      HEADERS[3] => -> (v) { v.to_f.round(3) },  # :low_price
+      HEADERS[4] => -> (v) { v.to_f.round(3) },  # :close_price
+      # HEADERS[5] - :adj_close_price doesn't exist in Alpha Vantage CSV
+      HEADERS[6] => -> (v) { v.to_i }            # :volume
     }
 
     ################################################################
 
     # Get recent data from JSON API
-    #
     # ticker String the security to retrieve
-    # returns a DataFrame
-    #
-    # NOTE: The function=TIME_SERIES_DAILY_ADJUSTED
-    #       is not a free API endpoint from Alpha Vantange.
-    #       So we are just using the free API endpoint
-    #       function=TIME_SERIES_DAILY
-    #       This means that we are not getting the
-    #       real adjusted closing price.  To sync
-    #       the columns with those from Yahoo Finance
-    #       we are duplicating the unadjusted clossing price
-    #       and adding that to the data frame as if it were
-    #       adjusted.
-    #
+    # returns a Polars DataFrame
     def self.recent(ticker, full: false, from_date: nil)
-
-      # NOTE: Using the CSV format because the JSON format has
-      #       really silly key values.  The column names for the
-      #       CSV format are much better.
       response  = CONNECTION.get(
         "/query?" +
         "function=TIME_SERIES_DAILY&" +
@@ -68,41 +52,35 @@ class SQA::DataFrame
         raise "Bad Response: #{response[:status]}"
       end
 
-      raw           = response[:body].split
-      headers       = raw.shift.split(',')
-
-      headers[0]    = 'date'  # website returns "timestamp" but that
-                              # has an unintended side-effect when
-                              # the names are normalized.
-                              # SMELL: IS THIS STILL TRUE?
-
-      close_inx     = headers.size - 2
-      adj_close_inx = close_inx + 1
-
-      headers.insert(adj_close_inx, 'adjusted_close')
-
-      aofh    = raw.map do |e|
-                  e2 = e.split(',')
-                  e2[1..-2] = e2[1..-2].map(&:to_f) # converting open, high, low, close
-                  e2[-1]    = e2[-1].to_i           # converting volumn
-                  e2.insert(adj_close_inx, e2[close_inx]) # duplicate the close price as a fake adj close price
-                  headers.zip(e2).to_h
-                end
-
+      # Read CSV into Polars DataFrame directly
+      df = Polars.read_csv(
+        StringIO.new(response[:body]),
+        dtypes: {
+          "open" => :f64,
+          "high" => :f64,
+          "low" => :f64,
+          "close" => :f64,
+          "volume" => :i64
+        }
+      )
+
+      # Handle date criteria if applicable
       if from_date
-        aofh.reject!{|e| Date.parse(e['date']) < from_date}
+        # Use Polars.col() to create an expression for filtering
+        df = df.filter(Polars.col("timestamp") >= from_date.to_s)
       end
 
-      return nil if aofh.empty?
+      # Wrap in SQA::DataFrame with proper transformers
+      # Note: mapping is applied first (renames columns), then transformers
+      sqa_df = SQA::DataFrame.new(df, transformers: TRANSFORMERS, mapping: HEADER_MAPPING)
 
-      # ensure tha the data frame is
-      # always sorted oldest to newest.
-
-      if aofh.first['date'] > aofh.last['date']
-        aofh.reverse!
-      end
+      # Alpha Vantage doesn't split close/adjusted_close, so duplicate for compatibility
+      # This ensures adj_close_price exists for strategies that expect it
+      sqa_df.data = sqa_df.data.with_column(
+        sqa_df.data["close_price"].alias("adj_close_price")
+      )
 
-      SQA::DataFrame.from_aofh(aofh, mapping: HEADER_MAPPING, transformers: TRANSFORMERS)
+      sqa_df
     end
   end
 end

data/lib/sqa/data_frame/data.rb

@@ -0,0 +1,92 @@
+# lib/sqa/data_frame/data.rb
+# frozen_string_literal: true
+
+require 'json'
+
+class SQA::DataFrame
+  # Data class to store stock metadata
+  #
+  # This class holds metadata about a stock including its ticker symbol,
+  # name, exchange, data source, technical indicators, and company overview.
+  #
+  # @example Creating from named parameters
+  #   data = SQA::DataFrame::Data.new(
+  #     ticker: 'AAPL',
+  #     source: :alpha_vantage,
+  #     indicators: { rsi: [30, 70], sma: [20, 50] }
+  #   )
+  #
+  # @example Creating from hash (JSON)
+  #   json_data = JSON.parse(File.read('stock_data.json'))
+  #   data = SQA::DataFrame::Data.new(json_data)
+  #
+  class Data
+    attr_accessor :ticker, :name, :exchange, :source, :indicators, :overview
+
+    # Initialize stock metadata
+    #
+    # Can be called in two ways:
+    #   1. With a hash: SQA::DataFrame::Data.new(hash) - for JSON deserialization
+    #   2. With keyword args: SQA::DataFrame::Data.new(ticker: 'AAPL', source: :alpha_vantage, ...)
+    #
+    # @param data_hash [Hash, nil] Hash of all attributes (when passed as first positional arg)
+    # @param ticker [String, Symbol, nil] Ticker symbol
+    # @param name [String, nil] Stock name
+    # @param exchange [String, nil] Exchange symbol (e.g., 'NASDAQ', 'NYSE')
+    # @param source [Symbol] Data source (:alpha_vantage, :yahoo_finance)
+    # @param indicators [Hash] Technical indicators configuration
+    # @param overview [Hash] Company overview data
+    #
+    def initialize(data_hash = nil, ticker: nil, name: nil, exchange: nil, source: :alpha_vantage, indicators: {}, overview: {})
+      if data_hash.is_a?(Hash) && ticker.nil?
+        # Initialize from hash (JSON deserialization) - called as: new(hash)
+        @ticker = data_hash['ticker'] || data_hash[:ticker]
+        @name = data_hash['name'] || data_hash[:name]
+        @exchange = data_hash['exchange'] || data_hash[:exchange]
+        @source = data_hash['source'] || data_hash[:source] || source
+        @indicators = data_hash['indicators'] || data_hash[:indicators] || {}
+        @overview = data_hash['overview'] || data_hash[:overview] || {}
+
+        # Convert source to symbol if it's a string
+        @source = @source.to_sym if @source.is_a?(String)
+      else
+        # Initialize from named parameters - called as: new(ticker: 'AAPL', ...)
+        @ticker = ticker
+        @name = name
+        @exchange = exchange
+        @source = source
+        @indicators = indicators
+        @overview = overview
+      end
+    end
+
+    # Serialize to JSON string
+    #
+    # @return [String] JSON representation
+    def to_json(*args)
+      to_h.to_json(*args)
+    end
+
+    # Convert to hash
+    #
+    # @return [Hash] Hash representation
+    def to_h
+      {
+        ticker: @ticker,
+        name: @name,
+        exchange: @exchange,
+        source: @source,
+        indicators: @indicators,
+        overview: @overview
+      }
+    end
+
+    # String representation
+    #
+    # @return [String] Human-readable representation
+    def to_s
+      "#{@ticker || 'Unknown'} (#{@exchange || 'N/A'}) via #{@source}"
+    end
+    alias_method :inspect, :to_s
+  end
+end

data/lib/sqa/data_frame/yahoo_finance.rb

@@ -1,13 +1,14 @@
 # lib/sqa/data_frame/yahoo_finance.rb
 # frozen_string_literal: true
 
+require 'polars'
+
 =begin
   The website financial.yahoo.com no longer supports an API.
   To get recent stock historical price updates you have
   to scrape the webpage.
 =end
 
-
 class SQA::DataFrame
   class YahooFinance
     CONNECTION  = Faraday.new(url: 'https://finance.yahoo.com')
@@ -21,60 +22,51 @@ class SQA::DataFrame
                     :volume,          # 6
                   ]
 
-      # The Yahoo Finance Headers are being remapped so that
-      # the header can be used as a method name to access the
-      # vector.
-      #
-      HEADER_MAPPING = {
-        "Date"      => HEADERS[0],
-        "Open"      => HEADERS[1],
-        "High"      => HEADERS[2],
-        "Low"       => HEADERS[3],
-        "Close"     => HEADERS[4],
-        "Adj Close" => HEADERS[5],
-        "Volume"    => HEADERS[6]
-      }
+    HEADER_MAPPING = {
+      "Date"      => HEADERS[0],
+      "Open"      => HEADERS[1],
+      "High"      => HEADERS[2],
+      "Low"       => HEADERS[3],
+      "Close"     => HEADERS[4],
+      "Adj Close" => HEADERS[5],
+      "Volume"    => HEADERS[6]
+    }
 
     ################################################################
 
-
     # Scrape the Yahoo Finance website to get recent
     # historical prices for a specific ticker
-    #
-    # ticker String the security to retrieve
-    # returns a DataFrame
-    #
+    # returns a Polars DataFrame
     def self.recent(ticker)
-      response  = CONNECTION.get("/quote/#{ticker.upcase}/history")
-      doc       = Nokogiri::HTML(response.body)
-      table     = doc.css('table').first
+      response = CONNECTION.get("/quote/#{ticker.upcase}/history")
+      doc = Nokogiri::HTML(response.body)
+      table = doc.css('table').first
 
       raise "NoDataError" if table.nil?
 
-      rows      = table.css('tbody tr')
-
-      aofh = []
+      rows = table.css('tbody tr')
 
-      rows.each do |row|
-        cols = row.css('td').map{|c| c.children[0].text}
+      data = rows.map do |row|
+        cols = row.css('td').map { |c| c.children[0].text }
 
-        next unless 7 == cols.size
+        next unless cols.size == 7
         next if cols[1]&.include?("Dividend")
-
-        if cols.any?(nil)
-          debug_me('== ERROR =='){[
-            :cols
-          ]}
-          next
-        end
-
-        cols[0] = Date.parse(cols[0]).to_s
-        cols[6] = cols[6].tr(',','').to_i
-        (1..5).each {|x| cols[x] = cols[x].to_f}
-        aofh << HEADERS.zip(cols).to_h
-      end
-
-      aofh
+        next if cols.any?(nil)
+
+        {
+          "Date"      => Date.parse(cols[0]).to_s,
+          "Open"      => cols[1].to_f,
+          "High"      => cols[2].to_f,
+          "Low"       => cols[3].to_f,
+          "Close"     => cols[4].to_f,
+          "Adj Close" => cols[5].to_f,
+          "Volume"    => cols[6].tr(',', '').to_i
+        }
+      end.compact
+
+      # Create Polars DataFrame then wrap in SQA::DataFrame
+      polars_df = Polars::DataFrame.new(data)
+      SQA::DataFrame.new(polars_df, mapping: HEADER_MAPPING)
     end
   end
 end