sqa 0.0.24 → 0.0.32

data/lib/sqa/data_frame.rb

@@ -2,204 +2,162 @@
 # frozen_string_literal: true
 
 require 'forwardable'
+require 'csv'
+require 'polars'
 
+
+require_relative 'data_frame/data'
 require_relative 'data_frame/yahoo_finance'
 require_relative 'data_frame/alpha_vantage'
 
 class SQA::DataFrame
-  class Data < Hashie::Mash
-    # SNELL: Are all of these needed?
-    include Hashie::Extensions::Mash::KeepOriginalKeys
-    # include Hashie::Extensions::Mash::PermissiveRespondTo
-    include Hashie::Extensions::Mash::SafeAssignment
-    include Hashie::Extensions::Mash::SymbolizeKeys
-    # include Hashie::Extensions::Mash::DefineAccessors
-  end
-
   extend Forwardable
 
-  # @data is of class Data
   attr_accessor :data
 
-  # Expects a Hash of Arrays (hofa)
-  # mapping: and transformers: are optional
-  # mapping is a Hash { old_key => new_key }
-  # transformers is also a Hash { key => Proc}
-  def initialize(
-      raw_data=     {}, # Array of Hashes or hash of array or hash
-      mapping:      {}, # { old_key => new_key }
-      transformers: {}  # { key => Proc }
-    )
+  def initialize(raw_data = nil, mapping: {}, transformers: {})
+    @data = Polars::DataFrame.new(raw_data || [])
 
-    if raw_data.is_a? Hash
-      initialize_hofa(raw_data, mapping: mapping)
+    # IMPORTANT: Rename columns FIRST, then apply transformers
+    # Transformers expect renamed column names
+    rename_columns!(mapping) unless mapping.empty?
+    apply_transformers!(transformers) unless transformers.empty?
+  end
 
-    elsif raw_data.is_a?(Array)     &&
-          raw_data.first.is_a?(Hash)
-      initialize_aofh(raw_data, mapping: mapping)
 
-    else
-      raise BadParameterError, "Expecting Hash or Array of Hashes got: #{aofh_or_hofa.class}"
+  def apply_transformers!(transformers)
+    transformers.each do |col, transformer|
+      col_name = col.to_s
+      @data = @data.with_column(
+        @data[col_name].apply(&transformer).alias(col_name)
+      )
     end
-
-    coerce_vectors!(transformers) if good_data? && !(transformers.nil? || transformers.empty?)
   end
 
 
-  def good_data?
-    return false if @data.empty? || @data.values.all?{|v| v.nil? || v.empty?}
+  def rename_columns!(mapping)
+    # Normalize mapping keys to strings for consistent lookup
+    # mapping can have string or symbol keys, columns are always strings
+    string_mapping = mapping.transform_keys(&:to_s)
 
-    true
-  end
+    rename_mapping = @data.columns.each_with_index.map do |col, _|
+      # Try exact match first, then lowercase match
+      new_name = string_mapping[col] || string_mapping[col.downcase] || col
+      # Polars requires both keys and values to be strings
+      [col, new_name.to_s]
+    end.to_h
 
+    @data = @data.rename(rename_mapping)
+  end
 
-  def initialize_aofh(aofh, mapping:)
-    klass = self.class
 
-    hofa  = klass.aofh_to_hofa(
-                  aofh,
-                  mapping:  mapping
-                )
+  def append!(other_df)
+    self_row_count = @data.shape[0]
+    other_row_count = other_df.data.shape[0]
 
-    initialize_hofa(hofa, mapping: mapping)
-  end
+    @data = if self_row_count == 0
+              other_df.data
+            else
+              @data.vstack(other_df.data)
+            end
 
+    post_append_row_count = @data.shape[0]
+    expected_row_count = self_row_count + other_row_count
+    return if post_append_row_count == expected_row_count
 
-  def initialize_hofa(hofa, mapping:)
-    klass = self.class
-    hofa  = klass.normalize_keys(
-              hofa,
-              adapter_mapping: mapping
-            ) unless mapping.empty?
+    raise "Append Error: expected #{expected_row_count}, got #{post_append_row_count} "
 
-    @data = Data.new(hofa)
   end
+  alias concat! append!
 
+  # Concatenate another DataFrame, remove duplicates, and sort
+  # This is the preferred method for updating CSV data to prevent duplicates
+  #
+  # @param other_df [SQA::DataFrame] DataFrame to append
+  # @param sort_column [String] Column to use for deduplication and sorting (default: "timestamp")
+  # @param descending [Boolean] Sort order - false for ascending (oldest first, TA-Lib compatible), true for descending
+  def concat_and_deduplicate!(other_df, sort_column: "timestamp", descending: false)
+    # Concatenate the dataframes
+    @data = if @data.shape[0] == 0
+              other_df.data
+            else
+              @data.vstack(other_df.data)
+            end
 
+    # Remove duplicates based on sort_column, keeping first occurrence
+    @data = @data.unique(subset: [sort_column], keep: "first")
 
-  def to_csv(path_to_file)
-    CSV.open(path_to_file, 'w') do |csv|
-      csv << keys
-      size.times do |x|
-        csv << row(x)
-      end
-    end
+    # Sort by the specified column (Polars uses 'reverse' for descending)
+    @data = @data.sort(sort_column, reverse: descending)
   end
 
-
-  def to_json(path_to_file)
-    NotImplemented.raise
+  def columns
+    @data.columns
   end
 
 
-  def to_aofh
-    NotImplemented.raise
+  def keys
+    @data.columns
   end
+  alias vectors keys
 
-
-  def_delegator :@data, :to_h, :to_hofa
-  alias_method :to_h, :to_hofa
-
-
-  # The number of data rows
-  def size
-    data[@data.keys[0]].size
+  def to_h
+    @data.columns.map { |col| [col.to_sym, @data[col].to_a] }.to_h
   end
-  alias_method :nrows,  :size
-  alias_method :length, :size
-
 
-  def_delegator :@data,  :keys
-  alias_method :vectors, :keys
-  alias_method :columns, :keys
 
-
-  def ncols
-    keys.size
+  def to_csv(path_to_file)
+    @data.write_csv(path_to_file)
   end
 
 
-  def_delegator :@data, :values, :values
-  def_delegator :@data, :[], :[]
-  def_delegator :@data, :[]=, :[]=
-
-
-  # same as values.transpose
-  # TODO: do benchmark to see if the transpose method if faster
-  def rows
-    result = []
-    size.times do |x|
-      entry = row(x)
-      result << entry
-    end
-    result
+  def size
+    @data.height
   end
-  alias_method :to_a, :rows
-
-
-  def row(x)
-    if x.is_a?(Integer)
-      raise BadParameterError if x < 0 || x >= size
+  alias nrows size
+  alias length size
 
-    elsif x.is_a?(Hash)
-      raise BadParameterError, "x is #{x}" if x.size > 1
-      key = x.keys[0]
-      x   = @data[key].index(x[key])
-      raise BadParameterError, 'Not Found #{x}' if x.nil?
-      return keys.zip(row(x)).to_h
-
-    else
-      raise BadParameterError, "Unknown x.class: #{x.class}"
-    end
-
-    entry = []
-
-    keys.each do |key|
-      entry << @data[key][x]
-    end
-
-    entry
+  def ncols
+    @data.width
   end
 
 
-  def append!(new_df)
-    raise(BadParameterError, "Key mismatch") if keys != new_df.keys
-
-    keys.each do |key|
-      @data[key] += new_df[key]
-    end
+  # FPL Analysis - Calculate Future Period Loss/Profit
+  #
+  # @param column [String, Symbol] Column name containing prices (default: "adj_close_price")
+  # @param fpop [Integer] Future Period of Performance (days to look ahead)
+  # @return [Array<Array<Float, Float>>] Array of [min_delta, max_delta] pairs
+  #
+  # @example
+  #   stock = SQA::Stock.new(ticker: 'AAPL')
+  #   fpl_data = stock.df.fpl(fpop: 10)
+  #
+  def fpl(column: 'adj_close_price', fpop: 14)
+    prices = @data[column.to_s].to_a
+    SQA::FPOP.fpl(prices, fpop: fpop)
   end
-  alias_method :concat!, :append!
 
 
-  # Creates a new instance with new keys
-  # based on the mapping hash where
-  #   { old_key => new_key }
+  # FPL Analysis with risk metrics and classification
   #
-  def rename(mapping)
-    SQA::DataFrame.new(
-      self.class.rename(
-        mapping,
-        @data.to_h
-      )
-    )
+  # @param column [String, Symbol] Column name containing prices (default: "adj_close_price")
+  # @param fpop [Integer] Future Period of Performance
+  # @return [Array<Hash>] Array of analysis hashes
+  #
+  # @example
+  #   analysis = stock.df.fpl_analysis(fpop: 10)
+  #   analysis.first[:direction]  # => :UP, :DOWN, :UNCERTAIN, or :FLAT
+  #   analysis.first[:magnitude]  # => Average expected movement percentage
+  #   analysis.first[:risk]       # => Volatility range
+  #
+  def fpl_analysis(column: 'adj_close_price', fpop: 14)
+    prices = @data[column.to_s].to_a
+    SQA::FPOP.fpl_analysis(prices, fpop: fpop)
   end
-  alias_method :rename_vectors, :rename
 
 
-  # Map the values of the vectors into different objects
-  # types is a Hash where the key is the vector name and
-  #   the value is a proc
-  #
-  # For Example:
-  # {
-  #   price: -> (v) {v.to_f.round(3)}
-  # }
-  #
-  def coerce_vectors!(transformers)
-    transformers.each_pair do |key, transformer|
-      @data[key].map!{|v| transformer.call(v)}
-    end
+  def self.is_date?(value)
+    value.is_a?(String) && !/\d{4}-\d{2}-\d{2}/.match(value).nil?
   end
 
 
@@ -219,135 +177,88 @@ class SQA::DataFrame
     @data.respond_to?(method_name) || super
   end
 
-  #################################################
   class << self
-
-    def concat(base_df, other_df)
-      base_df.concat!(other_df)
-    end
-
-
-    # TODO: The Data class has its own load which also supports
-    #       YAML by default.  Maybe this method should
-    #       make use of @data = Data.load(source)
+    # Load a DataFrame from a file source
+    # This is the primary method for loading persisted DataFrames
     #
-    def load(source:, mapping: {}, transformers:{})
-      file_type = source.extname[1..].downcase.to_sym
-
-      df  = if :csv == file_type
-              from_csv_file(source, mapping: mapping, transformers: transformers)
-            elsif :json == file_type
-              from_json_file(source, mapping: mapping, transformers: transformers)
-            else
-              raise BadParameterError, "unsupported file type: #{file_type}"
-            end
-
-      unless transformers.empty?
-        df.coerce_vectors!(transformers)
-      end
-
-      df
+    # @param source [String, Pathname] Path to CSV file
+    # @param transformers [Hash] Column transformations to apply (usually not needed for cached data)
+    # @param mapping [Hash] Column name mappings (usually not needed for cached data)
+    # @return [SQA::DataFrame] Loaded DataFrame
+    #
+    # Note: For cached CSV files, transformers and mapping should typically be empty
+    # since transformations were already applied when the data was first fetched.
+    # We only apply them if the CSV has old-format column names that need migration.
+    def load(source:, transformers: {}, mapping: {})
+      df = Polars.read_csv(source.to_s)
+
+      # Auto-detect if CSV needs migration (has old column names like "open" instead of "open_price")
+      # Only apply mapping if explicitly provided (for migration scenarios)
+      new(df, mapping: mapping, transformers: transformers)
     end
 
-
     def from_aofh(aofh, mapping: {}, transformers: {})
-      new(
-        aofh,
-        mapping: mapping,
-        transformers: transformers
+      aoh_sanitized = aofh.map { |entry| entry.transform_keys(&:to_s) }
+      columns = aoh_sanitized.first.keys
+      data = aoh_sanitized.map(&:values)
+      df = Polars::DataFrame.new(
+        data,
+        columns: columns
       )
+      new(df)
     end
 
 
     def from_csv_file(source, mapping: {}, transformers: {})
-      aofh = []
-
-      CSV.foreach(source, headers: true) do |row|
-        aofh << row.to_h
-      end
-
-      from_aofh(aofh, mapping: mapping, transformers: transformers)
+      df = Polars.read_csv(source)
+      new(df, mapping: mapping, transformers: transformers)
     end
 
 
     def from_json_file(source, mapping: {}, transformers: {})
-      aofh = JSON.parse(source.read)
-
+      aofh = JSON.parse(File.read(source)).map { |entry| entry.transform_keys(&:to_s) }
       from_aofh(aofh, mapping: mapping, transformers: transformers)
     end
 
 
-    # aofh -- Array of Hashes
-    # hofa -- Hash of Arrays
-    def aofh_to_hofa(aofh, mapping: {}, transformers: {})
-      hofa = {}
-      keys = aofh.first.keys
-
-      keys.each do |key|
-        hofa[key] = []
-      end
-
-      aofh.each do |entry|
-        keys.each do |key|
-          hofa[key] << entry[key]
-        end
+    def generate_mapping(keys)
+      keys.each_with_object({}) do |key, hash|
+        hash[key.to_s] = underscore_key(key.to_s)
       end
-
-      # SMELL: This might not be necessary
-      normalize_keys(hofa, adapter_mapping: mapping)
-    end
-
-
-    def normalize_keys(hofa, adapter_mapping: {})
-      hofa    = rename(adapter_mapping, hofa)
-      mapping = generate_mapping(hofa.keys)
-
-      rename(mapping, hofa)
     end
 
 
-    def rename(mapping, hofa)
-      mapping.each_pair do |old_key, new_key|
-        hofa[new_key] = hofa.delete(old_key) if hofa.has_key?(old_key)
-      end
-
-      hofa
-    end
-
-
-    def generate_mapping(keys)
-      mapping = {}
-
-      keys.each do |key|
-        mapping[key] = underscore_key(sanitize_key(key)) unless key.is_a?(Symbol)
-      end
-
-      mapping
+    def underscore_key(key)
+      key.to_s
+         .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+         .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+         .gsub(/[^a-zA-Z0-9]/, ' ')
+         .squeeze(' ')
+         .strip
+         .tr(' ', '_')
+         .downcase
+         .to_sym
     end
 
+    alias sanitize_key underscore_key
 
-    # returns a snake_case Symbol
-    def underscore_key(key)
-      key.to_s.gsub(/::/, '/').
-      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
-      gsub(/([a-z\d])([A-Z])/,'\1_\2').
-      tr("-", "_").
-      downcase.to_sym
+    def normalize_keys(hash, adapter_mapping: {})
+      hash = rename(hash, adapter_mapping) unless adapter_mapping.empty?
+      mapping = generate_mapping(hash.keys)
+      rename(hash, mapping)
     end
 
 
-    # removes punctuation and specal characters,
-    # replaces space with underscore.
-    def sanitize_key(key)
-      key.tr('.():/','').gsub(/^\d+.?\s/, "").tr(' ','_')
+    def rename(hash, mapping)
+      mapping.each { |old_key, new_key| hash[new_key] = hash.delete(old_key) if hash.key?(old_key) }
+      hash
     end
 
 
-    # returns true if key is in a date format
-    # like 2023-06-03
-    def is_date?(key)
-      !/(\d{4}-\d{2}-\d{2})/.match(key.to_s).nil?
+    def aofh_to_hofa(aofh, mapping: {}, transformers: {})
+      hofa = Hash.new { |h, k| h[k.downcase] = [] }
+      aofh.each { |entry| entry.each { |key, value| hofa[key.to_s.downcase] << value } }
+      hofa
     end
   end
 end
-