quandl_cassandra 0.0.1

data/lib/quandl/cassandra_models/column/read/collapse.rb

@@ -0,0 +1,64 @@
+class Quandl::Cassandra::Column::Read::Collapse < Quandl::Cassandra::Column::Read
+  
+  def perform
+    # assign
+    attributes[:collapse] = collapse
+    attributes[:frequency] = frequency
+    attributes[:column_frequencies] = column_frequencies
+    attributes[:column_collapses] = column_collapses
+  end
+  
+  def collapse
+    # collapse is the collapse
+    coll = attributes[:collapse]
+    # when pristine.frequency and collapse are equal, the source data is being requested
+    coll = :source if attributes[:pristine][:frequency] == attributes[:collapse]
+    # when the collapse is daily, source data is being requested
+    coll = :source if coll == :daily || coll.blank?
+    coll
+  end
+  
+  def frequency
+    # frequency is the given frequency
+    freq = attributes[:frequency]
+    # grab from database
+    freq = DatasetAttribute.where(id: id).pluck(:frequency).first if freq.blank? and id.present?
+    # fallback to collapse
+    freq = attributes[:collapse] if freq.blank?
+    # if freq is source, daily is being requested
+    freq = :daily if freq == :source
+    freq
+  end
+  
+  def column_frequencies
+    # default to passed value
+    freqs = attributes[:column_frequencies]
+    # fallback to collapse value
+    freqs = attributes[:column_ids].collect{ attributes[:frequency] } if freqs.blank?
+    freqs
+  end
+  
+  # INPUT   column_frequencies: [ :daily, :weekly, :monthly ], collapse: :monthly
+  # OUPUT   [ :monthly, :monthly, :source ]
+  
+  # INPUT   column_frequencies: [ :daily, :weekly, :monthly ], collapse: :source
+  # OUPUT   [ :source, :source, :source ]
+
+  # INPUT   column_frequencies: [ :daily, :monthly, :annual ], collapse: :monthly
+  # OUPUT   [ :monthly, :source, :source ]
+  
+  def column_collapses
+    attributes[:column_frequencies].collect do |frequency|
+      # is frequency less than collapse?
+      if Quandl::Operation::Collapse.collapses_greater_than( frequency ).include?( attributes[:collapse] )
+        # this column needs to be collapsed
+        attributes[:collapse]
+      # otherwise frequency is greater than or equal to collapse
+      else
+        # so collapse is not needed
+        :source
+      end
+    end
+  end
+  
+end

data/lib/quandl/cassandra_models/column/read/column.rb

@@ -0,0 +1,18 @@
+class Quandl::Cassandra::Column::Read::Column < Quandl::Cassandra::Column::Read
+  
+  def perform
+    # given an id, this is a dataset
+    attributes[:column_ids] ||= Quandl::Cassandra::Dataset.find(id).column_ids if id.present?
+    # given a column, this is requesting a specific column
+    attributes[:column_ids] = pluck_column_id if column.present?
+  end
+  
+  def pluck_column_id
+    [ attributes[:column_ids][ column ] ]
+  end
+  
+  def column
+    attributes[:column].present? ? attributes[:column] - 1 : nil
+  end
+  
+end

data/lib/quandl/cassandra_models/column/read/data_table.rb

@@ -0,0 +1,57 @@
+class Quandl::Cassandra::Column::Read::DataTable < Quandl::Cassandra::Column::Read
+  
+  def perform
+    attributes[:data] = quandl_data( attributes[:data] )
+  end
+  
+  
+  def quandl_data(data)
+    # init
+    data = Quandl::Data.new( data )
+    # post process
+    data = collapse(data)
+    data = transform(data)
+    data = trim(data)
+    data = sort( data )
+    data = row( data )
+    data = limit(data)
+    data
+  end
+  
+  def trim(data)
+    return data unless attributes[:data_table].present?
+    data = data.sort_order( :asc ) if attributes[:data_table][:trim_start] || attributes[:data_table][:trim_end]
+    data = data.trim_start( attributes[:data_table][:trim_start] ) if attributes[:data_table][:trim_start]
+    data = data.trim_end( attributes[:data_table][:trim_end] ) if attributes[:data_table][:trim_end]
+    data
+  end
+  
+  def sort(data)
+    data = data.sort_order( attributes[:order] )
+  end
+  
+  def collapse(data)
+    data = data.collapse( attributes[:collapse] ) if attributes[:collapse]
+    data
+  end
+  
+  def transform(data)
+    data = data.transform( attributes[:transform] ) if attributes[:transform]
+    data
+  end
+  
+  def limit(data)
+    data = data.limit( attributes[:pristine][:limit] ) if attributes[:pristine][:limit]
+    data
+  end
+  
+  def row(data)
+    # if a row was requested and the data count is greater than a single row
+    if attributes[:row].present? && data.count > 1
+      # grab the last row
+      data.data_array = [data.data_array[-1]]
+    end
+    data
+  end
+  
+end

data/lib/quandl/cassandra_models/column/read/offset.rb

@@ -0,0 +1,114 @@
+class Quandl::Cassandra::Column::Read::Offset < Quandl::Cassandra::Column::Read
+  
+  def perform
+    # apply offset
+    apply_offset_with_transform
+    apply_offset
+  end
+  
+  def apply_offset_with_transform
+    # rdiff_from needs the data from the current to offset
+    if attributes[:row].present? && attributes[:transform] == :rdiff_from
+      # limit the results by trim 
+      attributes[:trim_start] = trim_start.occurrences_of_frequency_ago( attributes[:row], collapse_with_frequency ).end_of_frequency(collapse_with_frequency).jd if order == :desc
+      attributes[:trim_end] = trim_end.occurrences_of_frequency_ahead( attributes[:row], collapse_with_frequency ).end_of_frequency(collapse_with_frequency).jd if order == :asc
+      # the query should not limit or offset the data
+      attributes[:limit] = nil
+      attributes[:offset] = nil
+    end
+  end
+  
+  def apply_offset
+    return unless attributes[:offset].present?
+    order == :asc ? apply_offset_asc : apply_offset_desc
+  end
+  
+  def apply_offset_asc
+    # calculate ranges
+    offset_start  = trim_end.occurrences_of_frequency_ahead( offset, collapse_with_frequency ).start_of_frequency(collapse_with_frequency)
+    offset_end    = offset_start.occurrences_of_frequency_ahead( accuracy_with_limit, collapse_with_frequency ).end_of_frequency( collapse_with_frequency ) if limit
+    # set trims
+    attributes[:trim_start]  = offset_start.jd
+    attributes[:trim_end]    = offset_end.jd  if limit
+  end
+  
+  def apply_offset_desc
+    # calculate ranges
+    offset_start  = trim_start.occurrences_of_frequency_ago( offset, collapse_with_frequency ).end_of_frequency(collapse_with_frequency)
+    offset_end    = offset_start.occurrences_of_frequency_ago( accuracy_with_limit, collapse_with_frequency ).start_of_frequency( collapse_with_frequency ) if limit
+    # set trims
+    attributes[:trim_start] = offset_end.jd  if limit
+    attributes[:trim_end]   = offset_start.jd
+  end
+  
+  def trim_start
+    return @trim_start if defined?(@trim_start)
+    times = column_ids.collect do |id|
+      Quandl::Cassandra::Column.limit(1).order("type DESC").where( id: id, type: collapse ).pluck(:time) 
+    end
+    max_time = times.flatten.max
+    @trim_start = Date.jd( max_time )
+  end
+  
+  def trim_end
+    return @trim_end if defined?(@trim_end)
+    times = column_ids.collect do |id|
+      Quandl::Cassandra::Column.limit(1).order("type ASC").where( id: id, type: collapse ).pluck(:time)
+    end
+    max_time = times.flatten.max
+    @trim_end = Date.jd( max_time )
+  end
+  
+  def accuracy_with_limit
+    awl = accuracy
+    awl = awl + limit - 1 if limit && limit > 0
+    awl
+  end
+  
+  def accuracy
+    # did the query include an accuracy?
+    accuracy = attributes[:accuracy]
+    # otherwise guess at the accuracy by collapse
+    if accuracy.blank?
+      accuracy = case frequency
+      when :daily     then 0
+      when :weekly    then 1
+      when :monthly   then 1
+      when :quarterly then 1
+      when :annual    then 0
+      else
+        3
+      end
+    end
+    accuracy
+  end
+  
+  def column_ids
+    attributes[:column_ids]
+  end
+  
+  def limit
+    @limit ||= attributes[:limit]
+  end
+  
+  def offset
+    attributes[:offset]
+  end
+  
+  def order
+    @order ||= attributes[:order] == :asc ? :asc : :desc
+  end
+  
+  def collapse_with_frequency
+    collapse == :source ? frequency : collapse
+  end
+  
+  def frequency
+    attributes[:frequency]
+  end
+  
+  def collapse
+    attributes[:collapse]
+  end
+  
+end

data/lib/quandl/cassandra_models/column/read/query.rb

@@ -0,0 +1,55 @@
+class Quandl::Cassandra::Column::Read::Query < Quandl::Cassandra::Column::Read
+  
+  def perform
+    # attrs to result hash
+    attributes[:data] = select_data
+  end
+  
+  def select_data
+    prepared = Quandl::Cassandra::Base.prepare( statement )
+    data = {}
+    attributes[:column_ids].each_with_index do | id, index |
+      # pluck column type from collapses
+      type = attributes[:column_collapses][index].to_s
+      # bind and execute query
+      rows = prepared.execute( id, type, :one )
+      # collect result
+      rows.each do |row|
+        data[row['time']] ||= Array.new( attributes[:column_ids].count )
+        data[row['time']][index] ||= row['value']
+      end
+    end
+    data.collect(&:flatten)
+    # # fire off the queries
+    # futures = []
+    # attrs[:column_ids].each_with_index do | id, index |
+    #   # pluck column type from collapses
+    #   type = attrs[:column_collapses][index].to_s
+    #   # bind and execute query
+    #   futures << Quandl::Cassandra::Base.connection.execute_async( statement.bind( id, type ) )
+    # end
+    # # collect the results
+    # futures.each_with_index do |future, column_index|
+    #   t1 = Time.now
+    #   rows = JavaDriver::ResultSet.new( future.get_uninterruptibly ).to_a
+    #   rows.each do |row|
+    #     data[row[0]] ||= Array.new(attrs[:column_ids].count)
+    #     data[row[0]][column_index] ||= row[1]
+    #   end
+    #   JCQL::CommonLogger.info "#{cql} (#{attrs[:column_ids][column_index]}) (#{t1.elapsed.microseconds}ms)"
+    # end
+    # data
+  end
+  
+  def statement
+    cql = "SELECT time,value FROM columns WHERE"
+    # cql += "ORDER"
+    cql += " time >= #{attributes[:trim_start]} AND " if attributes[:trim_start]
+    cql += " time <= #{attributes[:trim_end]} AND " if attributes[:trim_end]
+    cql += " id = ? AND type = ?"
+    cql += " ORDER BY type #{attributes[:order]}" if attributes[:order]
+    cql += " LIMIT #{attributes[:limit]}" if attributes[:limit]
+    cql
+  end
+  
+end

data/lib/quandl/cassandra_models/column/read/row.rb

@@ -0,0 +1,20 @@
+class Quandl::Cassandra::Column::Read::Row < Quandl::Cassandra::Column::Read
+
+  def perform
+    return unless attributes[:row].present?
+    # ensure integer
+    row = attributes[:row]
+    # negative value needs inversion
+    if row < 0
+      attributes[:order]   = :asc
+      attributes[:offset]  = (row * -1) - 1
+      attributes[:limit]   = 1
+    # postive value is an offset of current observation
+    else
+      attributes[:order]   = :desc
+      attributes[:offset]  = row
+      attributes[:limit]   = 1
+    end
+  end
+  
+end

data/lib/quandl/cassandra_models/column/read/transform.rb

@@ -0,0 +1,53 @@
+class Quandl::Cassandra::Column::Read::Transform < Quandl::Cassandra::Column::Read
+  
+  def perform
+    return unless attributes[:transform].present?
+    # apply
+    apply_rdiff
+    apply_cumul
+  end
+  
+  def apply_cumul
+    # rdiff_from needs the data from the current to offset
+    return unless transform?( :cumul ) && limit.present?    
+    # data table needs to handle the offset since cumul requires all data
+    attributes[:data_table] = Quandl::Cassandra::Column::Read::Offset.call(attributes)
+    # the query should not limit or offset the data
+    attributes[:row] = nil
+    attributes[:limit] = nil
+    attributes[:offset] = nil
+  end
+  
+  def apply_rdiff
+    return unless transform? :rdiff, :diff
+    # limit requires limit
+    attributes[:limit] = attributes[:limit] + 1 if attributes[:limit]
+    # trims should be increased by one
+    attributes[:trim_start] = trim_start.occurrences_of_frequency_ago( 1, attributes[:collapse] ).jd if trim_start.present? && order == :desc
+    attributes[:trim_end] = trim_end.occurrences_of_frequency_ahead( 1, attributes[:collapse] ).jd if trim_end.present? && order == :asc
+  end
+  
+  def transform?(*keys)
+    keys.each do |key|
+      return true if key == attributes[:transform]
+    end
+    false
+  end
+  
+  def trim_end
+    @trim_end ||= attributes[:trim_end].present? ? Date.jd( attributes[:trim_end] ) : nil
+  end
+  
+  def trim_start
+    @trim_start ||= attributes[:trim_start].present? ? Date.jd( attributes[:trim_start] ) : nil
+  end
+  
+  def limit
+    attributes[:limit]
+  end
+  
+  def order
+    @order ||= attributes[:order] == :asc ? :asc : :desc
+  end
+  
+end

data/lib/quandl/cassandra_models/column/read/type.rb

@@ -0,0 +1,25 @@
+class Quandl::Cassandra::Column::Read::Type < Quandl::Cassandra::Column::Read
+  
+  def perform
+    # enforce types
+    attributes.each do |key, value|
+      attributes[key] = enforce_type(key, value)
+    end
+    # retain pristine copy of attrs
+    attributes[:pristine] ||= attributes.clone
+  end
+  
+  def enforce_type(key, value)
+    case key
+    when :limit, :column, :trim_start, :trim_end, :offset, :accuracy, :row
+      return value.try(:to_i)
+    when :collapse, :transform, :frequency
+      return value.try(:to_sym)
+    when :order
+      return value.try(:to_sym) == :asc ? :asc : :desc
+    else
+      value
+    end
+  end
+  
+end

data/lib/quandl/cassandra_models/column/read.rb

@@ -0,0 +1,28 @@
+class Quandl::Cassandra::Column::Read < Quandl::Strategy::Strategize
+
+  require_relative 'read/collapse'
+  require_relative 'read/column'
+  require_relative 'read/data_table'
+  require_relative 'read/offset'
+  require_relative 'read/query'
+  require_relative 'read/row'
+  require_relative 'read/transform'
+  require_relative 'read/type'
+
+  define_attributes :id
+
+  def self.perform(attributes)
+    strategy = Quandl::Strategy.new( attributes ) do |c|
+      c.use Quandl::Cassandra::Column::Read::Type
+      c.use Quandl::Cassandra::Column::Read::Row
+      c.use Quandl::Cassandra::Column::Read::Column
+      c.use Quandl::Cassandra::Column::Read::Collapse
+      c.use Quandl::Cassandra::Column::Read::Transform
+      c.use Quandl::Cassandra::Column::Read::Offset
+      c.use Quandl::Cassandra::Column::Read::Query
+      c.use Quandl::Cassandra::Column::Read::DataTable
+    end
+    strategy.perform
+  end
+
+end