sqa 0.0.24 → 0.0.32

data/lib/sqa/sector_analyzer.rb

@@ -0,0 +1,300 @@
+# frozen_string_literal: true
+
+require 'kbs/blackboard'
+
+# SQA::SectorAnalyzer - Sector-based analysis with KBS blackboards
+#
+# Uses separate KBS blackboards for each stock sector to:
+# - Track sector-wide patterns
+# - Detect sector regime (bull/bear/rotation)
+# - Analyze cross-stock correlations
+# - Share pattern discoveries across sector
+#
+# Key Assumption: Stocks in the same sector tend to move together
+#
+# Example:
+#   analyzer = SQA::SectorAnalyzer.new
+#   analyzer.add_stock('AAPL', sector: :technology)
+#   analyzer.add_stock('MSFT', sector: :technology)
+#
+#   # Discover patterns for entire tech sector
+#   patterns = analyzer.discover_sector_patterns(:technology)
+#
+# Sectors: :technology, :finance, :healthcare, :energy, :consumer, :industrial
+
+module SQA
+  class SectorAnalyzer
+    SECTORS = {
+      technology: %w[AAPL MSFT GOOGL NVDA AMD INTC],
+      finance: %w[JPM BAC GS MS C WFC],
+      healthcare: %w[JNJ UNH PFE ABBV TMO MRK],
+      energy: %w[XOM CVX COP SLB EOG MPC],
+      consumer: %w[AMZN TSLA HD WMT NKE MCD],
+      industrial: %w[CAT DE BA MMM HON UPS],
+      materials: %w[LIN APD SHW FCX NEM DD],
+      utilities: %w[NEE DUK SO D AEP EXC],
+      real_estate: %w[AMT PLD CCI EQIX SPG O],
+      communications: %w[META NFLX DIS CMCSA T VZ]
+    }.freeze
+
+    attr_reader :blackboards, :stocks_by_sector
+
+    def initialize(db_dir: '/tmp/sqa_sectors')
+      @blackboards = {}
+      @stocks_by_sector = Hash.new { |h, k| h[k] = [] }
+      @db_dir = db_dir
+
+      # Create directory for blackboard databases
+      require 'fileutils'
+      FileUtils.mkdir_p(@db_dir)
+
+      # Initialize blackboard for each sector
+      SECTORS.keys.each do |sector|
+        init_sector_blackboard(sector)
+      end
+    end
+
+    # Add a stock to sector analysis
+    #
+    # @param stock [SQA::Stock, String] Stock object or ticker
+    # @param sector [Symbol] Sector classification
+    #
+    def add_stock(stock, sector:)
+      raise ArgumentError, "Unknown sector: #{sector}" unless SECTORS.key?(sector)
+
+      ticker = stock.is_a?(String) ? stock : stock.ticker
+      @stocks_by_sector[sector] << ticker unless @stocks_by_sector[sector].include?(ticker)
+
+      # Assert fact in sector blackboard
+      kb = @blackboards[sector]
+      kb.add_fact(:stock_registered, {
+        ticker: ticker,
+        sector: sector,
+        registered_at: Time.now.to_i
+      })
+
+      ticker
+    end
+
+    # Discover patterns for an entire sector
+    #
+    # @param sector [Symbol] Sector to analyze
+    # @param stocks [Array<SQA::Stock>] Stock objects to analyze
+    # @param options [Hash] Pattern discovery options
+    # @return [Array<Hash>] Sector-wide patterns
+    #
+    def discover_sector_patterns(sector, stocks, **options)
+      raise ArgumentError, "Unknown sector: #{sector}" unless SECTORS.key?(sector)
+
+      kb = @blackboards[sector]
+      all_patterns = []
+
+      puts "=" * 70
+      puts "Discovering patterns for #{sector.to_s.upcase} sector"
+      puts "Analyzing #{stocks.size} stocks: #{stocks.map(&:ticker).join(', ')}"
+      puts "=" * 70
+      puts
+
+      # Discover patterns for each stock
+      stocks.each do |stock|
+        puts "\nAnalyzing #{stock.ticker}..."
+
+        generator = SQA::StrategyGenerator.new(stock: stock, **options)
+        patterns = generator.discover_patterns
+
+        # Assert pattern facts in blackboard
+        patterns.each_with_index do |pattern, i|
+          kb.add_fact(:pattern_discovered, {
+            ticker: stock.ticker,
+            pattern_id: "#{stock.ticker}_#{i}",
+            conditions: pattern.conditions,
+            frequency: pattern.frequency,
+            avg_gain: pattern.avg_gain,
+            sector: sector
+          })
+
+          all_patterns << {
+            ticker: stock.ticker,
+            pattern: pattern,
+            sector: sector
+          }
+        end
+      end
+
+      # Detect sector-wide patterns (patterns that appear in multiple stocks)
+      sector_patterns = find_common_patterns(all_patterns)
+
+      # Assert sector-wide patterns
+      sector_patterns.each do |sp|
+        kb.add_fact(:sector_pattern, {
+          sector: sector,
+          conditions: sp[:conditions],
+          stock_count: sp[:stocks].size,
+          stocks: sp[:stocks],
+          avg_frequency: sp[:avg_frequency],
+          avg_gain: sp[:avg_gain]
+        })
+      end
+
+      puts "\n" + "=" * 70
+      puts "Sector Analysis Complete"
+      puts "  Individual patterns found: #{all_patterns.size}"
+      puts "  Sector-wide patterns: #{sector_patterns.size}"
+      puts "=" * 70
+
+      sector_patterns
+    end
+
+    # Detect sector regime
+    #
+    # @param sector [Symbol] Sector to analyze
+    # @param stocks [Array<SQA::Stock>] Stock objects
+    # @return [Hash] Sector regime information
+    #
+    def detect_sector_regime(sector, stocks)
+      raise ArgumentError, "Unknown sector: #{sector}" unless SECTORS.key?(sector)
+
+      kb = @blackboards[sector]
+
+      # Detect regime for each stock
+      regimes = stocks.map do |stock|
+        SQA::MarketRegime.detect(stock)
+      end
+
+      # Determine consensus regime
+      regime_counts = regimes.group_by { |r| r[:type] }.transform_values(&:size)
+      consensus_regime = regime_counts.max_by { |_k, v| v }&.first
+
+      # Calculate sector strength (% of stocks in bull regime)
+      bull_count = regimes.count { |r| r[:type] == :bull }
+      sector_strength = (bull_count.to_f / stocks.size * 100).round(2)
+
+      result = {
+        sector: sector,
+        consensus_regime: consensus_regime,
+        sector_strength: sector_strength,
+        stock_regimes: regimes,
+        detected_at: Time.now
+      }
+
+      # Assert sector regime fact
+      kb.add_fact(:sector_regime, {
+        sector: sector,
+        regime: consensus_regime,
+        strength: sector_strength,
+        timestamp: Time.now.to_i
+      })
+
+      result
+    end
+
+    # Query sector blackboard
+    #
+    # @param sector [Symbol] Sector to query
+    # @param fact_type [Symbol] Type of fact to query
+    # @param pattern [Hash] Pattern to match
+    # @return [Array<KBS::Fact>] Matching facts
+    #
+    def query_sector(sector, fact_type, pattern = {})
+      kb = @blackboards[sector]
+      kb.working_memory.facts.select do |fact|
+        next false unless fact.type == fact_type
+        pattern.all? { |key, value| fact.attributes[key] == value }
+      end
+    end
+
+    # Print sector summary
+    #
+    # @param sector [Symbol] Sector to summarize
+    #
+    def print_sector_summary(sector)
+      kb = @blackboards[sector]
+
+      puts "\n" + "=" * 70
+      puts "#{sector.to_s.upcase} SECTOR SUMMARY"
+      puts "=" * 70
+
+      # Count facts by type
+      fact_counts = kb.working_memory.facts.group_by(&:type).transform_values(&:size)
+
+      puts "\nFacts in Blackboard:"
+      fact_counts.each do |type, count|
+        puts "  #{type}: #{count}"
+      end
+
+      # Show sector regime if available
+      regime_facts = query_sector(sector, :sector_regime)
+      if regime_facts.any?
+        latest = regime_facts.last
+        puts "\nCurrent Sector Regime:"
+        puts "  Type: #{latest[:regime]}"
+        puts "  Strength: #{latest[:strength]}%"
+      end
+
+      # Show sector patterns if available
+      pattern_facts = query_sector(sector, :sector_pattern)
+      if pattern_facts.any?
+        puts "\nSector-Wide Patterns: #{pattern_facts.size}"
+        pattern_facts.first(3).each_with_index do |fact, i|
+          puts "  #{i + 1}. Conditions: #{fact[:conditions]}"
+          puts "     Stocks: #{fact[:stocks].join(', ')}"
+          puts "     Avg Gain: #{fact[:avg_gain].round(2)}%"
+        end
+      end
+
+      puts "=" * 70
+    end
+
+    private
+
+    # Initialize KBS blackboard for a sector
+    def init_sector_blackboard(sector)
+      db_path = File.join(@db_dir, "#{sector}.db")
+
+      # Create blackboard with persistent storage
+      @blackboards[sector] = KBS::Blackboard::Engine.new(db_path: db_path)
+
+      # Define sector-specific rules
+      define_sector_rules(sector)
+    end
+
+    # Define rules for sector analysis
+    def define_sector_rules(sector)
+      kb = @blackboards[sector]
+
+      # Rule: Detect sector strength
+      rule = KBS::Rule.new("#{sector}_strength_detection") do |r|
+        r.conditions = [
+          KBS::Condition.new(:stock_registered, { sector: sector }),
+          KBS::Condition.new(:pattern_discovered, { sector: sector })
+        ]
+
+        r.action = lambda do |facts, bindings|
+          # Could add logic here to assert derived facts
+        end
+      end
+
+      kb.add_rule(rule)
+    end
+
+    # Find patterns common across multiple stocks
+    def find_common_patterns(all_patterns)
+      # Group by similar conditions
+      grouped = all_patterns.group_by do |p|
+        p[:pattern].conditions.sort.to_h
+      end
+
+      # Find groups with multiple stocks
+      common = grouped.select { |_conditions, group| group.size >= 2 }
+
+      common.map do |conditions, group|
+        {
+          conditions: conditions,
+          stocks: group.map { |p| p[:ticker] }.uniq,
+          avg_frequency: group.map { |p| p[:pattern].frequency }.sum / group.size.to_f,
+          avg_gain: group.map { |p| p[:pattern].avg_gain }.sum / group.size.to_f
+        }
+      end
+    end
+  end
+end

data/lib/sqa/stock.rb

@@ -1,197 +1,201 @@
 # lib/sqa/stock.rb
 
-
-# SMELL:  SQA::Stock is now pretty coupled to the Alpha Vantage
-#         API service.  Should that stuff be extracted into a
-#         separate class and injected by the requiring program?
-
 class SQA::Stock
   extend Forwardable
 
   CONNECTION = Faraday.new(url: "https://www.alphavantage.co")
 
-  attr_accessor :data # General Info      -- SQA::DataFrame::Data
-  attr_accessor :df   # Historical Prices -- SQA::DataFrame::Data
-
-  attr_accessor :klass        # class of historical and current prices
-  attr_accessor :transformers # procs for changing column values from String to Numeric
-
-  # Holds the SQA::Strategy class name which seems to work
-  # the best for this stock.
-  attr_accessor :strategy  # TODO: make part of the @data object
+  attr_accessor :data, :df, :klass, :transformers, :strategy
 
+  def initialize(ticker:, source: :alpha_vantage)
+    @ticker = ticker.downcase
+    @source = source
 
-  def initialize(
-        ticker:,
-        source: :alpha_vantage
-      )
+    @data_path = SQA.data_dir + "#{@ticker}.json"
+    @df_path = SQA.data_dir + "#{@ticker}.csv"
 
-    @ticker     = ticker.downcase
-    @source     = source
-
-    raise "Invalid Ticker #{ticker}" unless SQA::Ticker.valid?(ticker)
+    # Validate ticker if validation data is available and cached data doesn't exist
+    unless @data_path.exist? && @df_path.exist?
+      unless SQA::Ticker.valid?(ticker)
+        warn "Warning: Ticker #{ticker} could not be validated. Proceeding anyway." if $VERBOSE
+      end
+    end
 
-    @data_path  = SQA.data_dir + "#{@ticker}.json"
-    @df_path    = SQA.data_dir + "#{@ticker}.csv"
+    @klass = "SQA::DataFrame::#{@source.to_s.camelize}".constantize
+    @transformers = "SQA::DataFrame::#{@source.to_s.camelize}::TRANSFORMERS".constantize
 
-    @klass         = "SQA::DataFrame::#{@source.to_s.camelize}".constantize
-    @transformers  = "SQA::DataFrame::#{@source.to_s.camelize}::TRANSFORMERS".constantize
+    load_or_create_data
+    update_the_dataframe
+  end
 
+  def load_or_create_data
     if @data_path.exist?
-      load
+      @data = SQA::DataFrame::Data.new(JSON.parse(@data_path.read))
     else
-      create
-      update
-      save
-    end
-
-    update_the_dataframe
-  end
+      # Create minimal data structure
+      create_data
 
+      # Try to fetch overview data, but don't fail if we can't
+      # This is optional metadata - we can work with just price data
+      update
 
-  def load
-    @data = SQA::DataFrame::Data.new(
-              JSON.parse(@data_path.read)
-            )
+      # Save whatever data we have (even if overview fetch failed)
+      save_data
+    end
   end
 
-
-  def create
-    @data =
-      SQA::DataFrame::Data.new(
-        {
-          ticker:       @ticker,
-          source:       @source,
-          indicators:   { xyzzy: "Magic" },
-        }
-      )
+  def create_data
+    @data = SQA::DataFrame::Data.new(ticker: @ticker, source: @source, indicators: { xyzzy: "Magic" })
   end
 
-
   def update
-    merge_overview
+    begin
+      merge_overview
+    rescue => e
+      # Log warning but don't fail - overview data is optional
+      # Common causes: rate limits, network issues, API errors
+      warn "Warning: Could not fetch overview data for #{@ticker} (#{e.class}: #{e.message}). Continuing without it."
+    end
   end
 
-
-  def save
-    @data_path.write @data.to_json
+  def save_data
+    @data_path.write(@data.to_json)
   end
 
-
-  def_delegator :@data, :ticker,      :ticker
-  def_delegator :@data, :name,        :name
-  def_delegator :@data, :exchange,    :exchange
-  def_delegator :@data, :source,      :source
-  def_delegator :@data, :indicators,  :indicators
-  def_delegator :@data, :indicators=, :indicators=
-  def_delegator :@data, :overview,    :overview
-
-
+  def_delegators :@data, :ticker, :name, :exchange, :source, :indicators, :indicators=, :overview
 
   def update_the_dataframe
     if @df_path.exist?
-      @df     = SQA::DataFrame.load(
-        source:       @df_path,
-        transformers: @transformers
-      )
+      # Load cached CSV - transformers already applied when data was first fetched
+      # Don't reapply them as columns are already in correct format
+      @df = SQA::DataFrame.load(source: @df_path)
+
+      migrated = false
+
+      # Migration 1: Rename old column names to new convention
+      # Old files may have: open, high, low, close
+      # New files should have: open_price, high_price, low_price, close_price
+      if @df.columns.include?("open") && !@df.columns.include?("open_price")
+        old_to_new_mapping = {
+          "open"   => "open_price",
+          "high"   => "high_price",
+          "low"    => "low_price",
+          "close"  => "close_price"
+        }
+        @df.rename_columns!(old_to_new_mapping)
+        migrated = true
+      end
+
+      # Migration 2: Add adj_close_price column if missing (for old cached files)
+      # This ensures compatibility when appending new data that includes this column
+      unless @df.columns.include?("adj_close_price")
+        @df.data = @df.data.with_column(
+          @df.data["close_price"].alias("adj_close_price")
+        )
+        migrated = true
+      end
+
+      # Save migrated DataFrame to avoid repeating migration
+      @df.to_csv(@df_path) if migrated
     else
-      @df     = klass.recent(@ticker, full: true)
-      @df.to_csv(@df_path)
-      return
+      # Fetch fresh data from source (applies transformers and mapping)
+      begin
+        @df = @klass.recent(@ticker, full: true)
+        @df.to_csv(@df_path)
+        return
+      rescue => e
+        # If we can't fetch data, raise a more helpful error
+        raise "Unable to fetch data for #{@ticker}. Please ensure API key is set or provide cached CSV file at #{@df_path}. Error: #{e.message}"
+      end
     end
 
-    from_date = Date.parse(@df.timestamp.last) + 1
-    df2       = klass.recent(@ticker, from_date: from_date)
+    update_dataframe_with_recent_data
+  end
 
-    return if df2.nil?  # CSV file is up to date.
+  def update_dataframe_with_recent_data
+    return unless should_update?
 
-    df_nrows  = @df.nrows
-    @df.append!(df2)
+    begin
+      # CSV is sorted ascending (oldest first, TA-Lib compatible), so .last gets the most recent date
+      from_date = Date.parse(@df["timestamp"].to_a.last)
+      df2 = @klass.recent(@ticker, from_date: from_date)
 
-    if @df.nrows > df_nrows
-      @df.to_csv(@df_path)
+      if df2 && (df2.size > 0)
+        # Use concat_and_deduplicate! to prevent duplicate timestamps and maintain ascending sort
+        @df.concat_and_deduplicate!(df2)
+        @df.to_csv(@df_path)
+      end
+    rescue => e
+      # Log warning but don't fail - we have cached data
+      # Common causes: rate limits, network issues, API errors
+      warn "Warning: Could not update #{@ticker} from API (#{e.class}: #{e.message}). Using cached data."
     end
   end
 
+  def should_update?
+    # Don't update if we're in lazy update mode
+    return false if SQA.config.lazy_update
+
+    # Don't update if we don't have an API key (only relevant for Alpha Vantage)
+    if @source == :alpha_vantage
+      begin
+        SQA.av_api_key
+      rescue
+        return false
+      end
+    end
+
+    # Don't update if CSV data is already current (last timestamp is today or later)
+    # This prevents unnecessary API calls when we already have today's data
+    if @df && @df.size > 0
+      begin
+        last_timestamp = Date.parse(@df["timestamp"].to_a.last)
+        return false if last_timestamp >= Date.today
+      rescue => e
+        # If we can't parse the date, assume we need to update
+        warn "Warning: Could not parse last timestamp for #{@ticker} (#{e.message}). Will attempt update." if $VERBOSE
+      end
+    end
+
+    true
+  end
 
   def to_s
-    "#{ticker} with #{@df.size} data points from #{@df.timestamp.first} to #{@df.timestamp.last}"
+    "#{ticker} with #{@df.size} data points from #{@df["timestamp"].to_a.first} to #{@df["timestamp"].to_a.last}"
   end
+  # Note: CSV data is stored in ascending chronological order (oldest to newest)
+  # This ensures compatibility with TA-Lib indicators which expect arrays in this order
   alias_method :inspect, :to_s
 
-
   def merge_overview
     temp = JSON.parse(
       CONNECTION.get("/query?function=OVERVIEW&symbol=#{ticker.upcase}&apikey=#{SQA.av.key}")
-        .to_hash[:body]
+      .to_hash[:body]
     )
 
     if temp.has_key? "Information"
       ApiError.raise(temp["Information"])
     end
 
-    # TODO: CamelCase hash keys look common in Alpha Vantage
-    #       JSON; look at making a special Hashie-based class
-    #       to convert the keys to normal Ruby standards.
-
     temp2 = {}
-
-    string_values = %w[ address asset_type cik country currency
-                        description dividend_date ex_dividend_date
-                        exchange fiscal_year_end industry latest_quarter
-                        name sector symbol
-                      ]
+    string_values = %w[address asset_type cik country currency description dividend_date ex_dividend_date exchange fiscal_year_end industry latest_quarter name sector symbol]
 
     temp.keys.each do |k|
-      new_k         = k.underscore
-      temp2[new_k]  = string_values.include?(new_k) ? temp[k] : temp[k].to_f
+      new_k = k.underscore
+      temp2[new_k] = string_values.include?(new_k) ? temp[k] : temp[k].to_f
     end
 
     @data.overview = temp2
   end
 
-
-  def associate_best_strategy(strategies)
-    best_strategy = nil
-    best_accuracy = 0
-
-    strategies.each do |strategy|
-      accuracy = evaluate_strategy(strategy)
-
-      if accuracy > best_accuracy
-        best_strategy = strategy
-        best_accuracy = accuracy
-      end
-    end
-
-    self.strategy = best_strategy
-  end
-
-
-  def evaluate_strategy(strategy)
-    # TODO: Implement this method to evaluate the accuracy of the strategy
-    #       on the historical data of this stock.
-  end
-
-
-
   #############################################
   ## Class Methods
 
   class << self
-    @@top = nil
-
-    # Top Gainers, Losers and Most Active for most
-    # recent closed trading day.
-    #
     def top
       return @@top unless @@top.nil?
 
-      a_hash  = JSON.parse(
-                  CONNECTION.get(
-                    "/query?function=TOP_GAINERS_LOSERS&apikey=#{SQA.av.key}"
-                  ).to_hash[:body]
-                )
+      a_hash = JSON.parse(CONNECTION.get("/query?function=TOP_GAINERS_LOSERS&apikey=#{SQA.av.key}").to_hash[:body])
 
       mash = Hashie::Mash.new(a_hash)
 

data/lib/sqa/strategy/bollinger_bands.rb

@@ -0,0 +1,42 @@
+# lib/sqa/strategy/bollinger_bands.rb
+# frozen_string_literal: true
+
+# Bollinger Bands trading strategy
+# Buy when price touches lower band (oversold)
+# Sell when price touches upper band (overbought)
+#
+class SQA::Strategy::BollingerBands
+  def self.trade(vector)
+    return :hold unless vector.respond_to?(:prices) && vector.prices&.size >= 20
+
+    prices = vector.prices
+    period = 20
+    std_dev = 2.0
+
+    # Calculate Bollinger Bands using SQAI
+    upper, middle, lower = SQAI.bbands(prices, period: period, nbdev_up: std_dev, nbdev_down: std_dev)
+
+    return :hold if upper.nil? || lower.nil?
+
+    current_price = prices.last
+    upper_band = upper.last
+    lower_band = lower.last
+    middle_band = middle.last
+
+    # Buy signal: price at or below lower band (oversold)
+    if current_price <= lower_band
+      :buy
+
+    # Sell signal: price at or above upper band (overbought)
+    elsif current_price >= upper_band
+      :sell
+
+    # Hold: price between bands
+    else
+      :hold
+    end
+  rescue => e
+    warn "BollingerBands strategy error: #{e.message}"
+    :hold
+  end
+end

data/lib/sqa/strategy/consensus.rb

@@ -26,9 +26,12 @@ class SQA::Strategy::Consensus
 	def consensus
 		count = @results.group_by(&:itself).transform_values(&:count)
 
-		if count[:buy] > count[:sell]
+		buy_count = count[:buy].to_i
+		sell_count = count[:sell].to_i
+
+		if buy_count > sell_count
 			:buy
-		elsif count[:sell] > count[:buy]
+		elsif sell_count > buy_count
 			:sell
 		else
 			:hold