quandl_cassandra_models 0.3.6

data/lib/quandl/cassandra/models/column/read/row.rb

@@ -0,0 +1,20 @@
+class Quandl::Cassandra::Models::Column::Read::Row < Quandl::Cassandra::Models::Column::Read
+
+  def perform
+    return unless attributes[:row].present?
+    # ensure integer
+    row = attributes[:row]
+    # negative value needs inversion
+    if row < 0
+      attributes[:order]   = :asc
+      attributes[:offset]  = (row * -1) - 1
+      attributes[:limit]   = 1
+    # postive value is an offset of current observation
+    else
+      attributes[:order]   = :desc
+      attributes[:offset]  = row
+      attributes[:limit]   = 1
+    end
+  end
+  
+end

data/lib/quandl/cassandra/models/column/read/select_columns.rb

@@ -0,0 +1,63 @@
+class Quandl::Cassandra::Models::Column::Read::SelectColumns < Quandl::Cassandra::Models::Column::Read
+  
+  def perform
+    # attrs to result hash
+    attributes[:data] = count? ? count_data : select_data
+  rescue => e
+    Quandl::Logger.error("#{attributes} #{e}")
+    raise
+  end
+  
+  def count_data
+    prepared = Quandl::Cassandra::Base.prepare( statement )
+    rows = []
+    column_ids.each_with_index do | id, index |
+      # pluck column type from collapses
+      type = attributes[:column_collapses][index].to_s
+      # bind and execute query
+      rows << prepared.execute( id, type, :one )
+    end
+    rows.collect{|r| r.first['count'] }.max
+  end
+  
+  def select_data
+    t1 = Time.now
+    # fire off the queries
+    prepared = Quandl::Cassandra::Base.prepare( statement )
+    data = {}
+    futures = []
+    column_ids.each_with_index do | id, index |
+      # pluck column type from collapses
+      type = attributes[:column_collapses][index].to_s
+      # bind and execute query
+      futures << prepared.async.execute( id, type, Quandl::Cassandra::Base.consistency )
+    end
+    # collect the results
+    futures.each_with_index do |future, index|
+      # collect result 
+      future.value.each do |row|
+        data[row['time']] ||= Array.new( column_ids.count )
+        data[row['time']][index] ||= row['value']
+      end
+    end
+    Quandl::Logger.debug("(#{t1.elapsed_ms}) #{self.class.name}.select_data")
+    data
+  end
+  
+  def statement
+    columns = count? ? "COUNT(*)" : "time,value"
+    cql = "SELECT #{columns} FROM columns WHERE"
+    # cql += "ORDER"
+    cql += " time >= #{attributes[:trim_start]} AND " if attributes[:trim_start]
+    cql += " time <= #{attributes[:trim_end]} AND " if attributes[:trim_end]
+    cql += " id = ? AND type = ?"
+    cql += " ORDER BY type #{order}"
+    cql += " LIMIT #{attributes[:limit]}" if attributes[:limit]
+    cql
+  end
+  
+  def order
+    @order ||= attributes[:order] == :asc ? :asc : :desc
+  end
+  
+end

data/lib/quandl/cassandra/models/column/read/transform.rb

@@ -0,0 +1,53 @@
+class Quandl::Cassandra::Models::Column::Read::Transform < Quandl::Cassandra::Models::Column::Read
+  
+  def perform
+    return unless attributes[:transform].present?
+    # apply
+    apply_rdiff
+    apply_cumul
+  end
+  
+  def apply_cumul
+    # rdiff_from needs the data from the current to offset
+    return unless transform?( :cumul ) && limit.present?    
+    # data table needs to handle the offset since cumul requires all data
+    attributes[:data_table] = Quandl::Cassandra::Models::Column::Read::Offset.call(attributes)
+    # the query should not limit or offset the data
+    attributes[:row] = nil
+    attributes[:limit] = nil
+    attributes[:offset] = nil
+  end
+  
+  def apply_rdiff
+    return unless transform? :rdiff, :diff
+    # limit requires limit
+    attributes[:limit] = attributes[:limit] + 1 if attributes[:limit]
+    # trims should be increased by one
+    attributes[:trim_start] = trim_start.occurrences_of_frequency_ago( 1, attributes[:collapse] ).jd if trim_start.present? && order == :desc
+    attributes[:trim_end] = trim_end.occurrences_of_frequency_ahead( 1, attributes[:collapse] ).jd if trim_end.present? && order == :asc
+  end
+  
+  def transform?(*keys)
+    keys.each do |key|
+      return true if key == attributes[:transform]
+    end
+    false
+  end
+  
+  def trim_end
+    @trim_end ||= attributes[:trim_end].present? ? Date.jd( attributes[:trim_end] ) : nil
+  end
+  
+  def trim_start
+    @trim_start ||= attributes[:trim_start].present? ? Date.jd( attributes[:trim_start] ) : nil
+  end
+  
+  def limit
+    attributes[:limit]
+  end
+  
+  def order
+    @order ||= attributes[:order] == :asc ? :asc : :desc
+  end
+  
+end

data/lib/quandl/cassandra/models/column/read/trim.rb

@@ -0,0 +1,14 @@
+class Quandl::Cassandra::Models::Column::Read::Trim < Quandl::Cassandra::Models::Column::Read
+  
+  def perform
+    # if both are provided
+    if attributes[:trim_end].is_a?(Integer) && attributes[:trim_start].is_a?(Integer)
+      # if trim_start exceeds trim_end
+      if attributes[:trim_start] > attributes[:trim_end]
+        # then set trim_end to trim_start, since they are requesting nothing
+        attributes[:trim_end] = attributes[:trim_start]
+      end
+    end
+  end
+  
+end

data/lib/quandl/cassandra/models/column/read/type.rb

@@ -0,0 +1,25 @@
+class Quandl::Cassandra::Models::Column::Read::Type < Quandl::Cassandra::Models::Column::Read
+  
+  def perform
+    # enforce types
+    attributes.each do |key, value|
+      attributes[key] = enforce_type(key, value)
+    end
+    # retain pristine copy of attrs
+    attributes[:pristine] ||= attributes.clone
+  end
+  
+  def enforce_type(key, value)
+    case key
+    when :limit, :column, :trim_start, :trim_end, :offset, :accuracy, :row
+      return value.try(:to_i)
+    when :collapse, :transform, :frequency
+      return value.try(:to_sym)
+    when :order
+      return value.try(:to_sym) == :asc ? :asc : :desc
+    else
+      value
+    end
+  end
+  
+end

data/lib/quandl/cassandra/models/column/write.rb

@@ -0,0 +1,25 @@
+class Quandl::Cassandra::Models::Column::Write < Quandl::Strategy::Strategize
+  
+  # strategy attributes
+  define_attributes :id, :data, :frequency, :column_ids, :frequency_data, :frequency_column_data, :statement_values
+  
+  require_relative 'write/insert_columns'
+  require_relative 'write/insert_column_attributes'
+  require_relative 'write/group_data_by_column'
+  require_relative 'write/group_data_by_frequency'
+  
+  # execute strategy
+  def self.perform(attributes)
+    strategy = Quandl::Strategy.new( attributes ) do |c|
+      c.use Quandl::Cassandra::Models::Column::Write::GroupDataByFrequency
+      c.use Quandl::Cassandra::Models::Column::Write::GroupDataByColumn
+      c.use Quandl::Cassandra::Models::Column::Write::InsertColumns
+      c.use Quandl::Cassandra::Models::Column::Write::InsertColumnAttributes
+    end
+    strategy.perform
+  rescue => e
+    Quandl::Logger.error("#{e} #{strategy.attributes}")
+    raise
+  end
+  
+end

data/lib/quandl/cassandra/models/column/write/group_data_by_column.rb

@@ -0,0 +1,36 @@
+class Quandl::Cassandra::Models::Column::Write::GroupDataByColumn < Quandl::Cassandra::Models::Column::Write
+  
+  # INPUTS
+  # { source: [ [1,2,3], [2,4,8], ... ], weekly: ... }
+  
+  def perform
+    group_by_statement_values
+  end
+  
+  def group_by_statement_values
+    # for each [ [date, val, val], ... ]
+    self.statement_values = []
+    frequency_data.each do |frequency, rows|
+      frequency = frequency.to_s
+      rows.each do |row|
+        # extract date
+        date = row[0]
+        # for each [ val, val, ... ]
+        row[1..-1].each_with_index do |value, index|
+          # ensure array
+          cid = column_id(index)
+          # add to values
+          self.statement_values << [ cid, frequency, date, value] unless value.blank?
+        end
+      end
+    end
+  end
+  
+  def column_id(index)
+    # ensure column_ids is defined
+    self.column_ids ||= Quandl::Cassandra::Models::Dataset.find_column_ids_by_id(id)
+    # ensure column_ids[index] is present
+    self.column_ids[index] ||= Cql::Uuid.new(SecureRandom.uuid)
+  end
+  
+end

data/lib/quandl/cassandra/models/column/write/group_data_by_frequency.rb

@@ -0,0 +1,24 @@
+class Quandl::Cassandra::Models::Column::Write::GroupDataByFrequency < Quandl::Cassandra::Models::Column::Write
+  
+  # INPUTS
+  # [ [1,2,3], [2,4,8], ... ]
+  
+  # OUTPUTS
+  # { source: [ [1,2,3], [2,4,8], ... ], weekly: ... }
+  
+  def perform
+    self.frequency = data.frequency
+    self.frequency_data = {}
+    # clone source data
+    self.data = data.clone
+    self.frequency_data[:source] = data.data_array.clone
+    # collapse and clone each frequency
+    Quandl::Operation::Collapse.collapses_greater_than(data.frequency).each do |freq|
+      # collapse the data to the required frequency
+      data.collapse(freq)
+      # clone the internal data array and add it to the grouping
+      self.frequency_data[freq] = data.data_array.clone
+    end
+  end
+  
+end

data/lib/quandl/cassandra/models/column/write/insert_column_attributes.rb

@@ -0,0 +1,22 @@
+class Quandl::Cassandra::Models::Column::Write::InsertColumnAttributes < Quandl::Cassandra::Models::Column::Write
+  
+  # INPUTS
+  # { source: { UUID: [[1,2], [2,4]], UUID: [[1,3],[2,8]] }}
+  
+  def perform
+    return if column_ids.blank?
+    # format data for batch insertion
+    rows_values = []
+    column_ids.each_with_index do |column_id, position|
+      rows_values << [id, column_id, position, frequency]
+    end
+    # insert dataset_attribute
+    Quandl::Cassandra::Base.execute("INSERT INTO dataset_attributes (id, updated_at, frequency) VALUES (#{id}, #{(Time.now.to_f * 1000).to_i}, '#{frequency}')")
+    # insert data
+    Quandl::Cassandra::Batch.insert(rows_values) do |id, column_id, position, frequency|
+      %Q{INSERT INTO datasets (id, column_id, position) VALUES (#{id}, #{column_id}, #{position})
+      INSERT INTO column_attributes ( id, frequency ) VALUES ( #{column_id}, '#{frequency}' )}
+    end
+  end
+  
+end

data/lib/quandl/cassandra/models/column/write/insert_columns.rb

@@ -0,0 +1,9 @@
+class Quandl::Cassandra::Models::Column::Write::InsertColumns < Quandl::Cassandra::Models::Column::Write
+  
+  def perform
+    Quandl::Cassandra::Batch.insert(statement_values) do |id, type, time, value|
+      "INSERT INTO columns (id, type, time, value) VALUES (#{id}, '#{type}', #{time}, #{value})"
+    end
+  end
+  
+end

data/lib/quandl/cassandra/models/column_attribute.rb

@@ -0,0 +1,11 @@
+class Quandl::Cassandra::Models::ColumnAttribute < Quandl::Cassandra::Base
+
+  table_name :column_attributes
+
+  after_initialize :default_attributes
+
+  def default_attributes
+    self.id = Cql::Uuid.new(SecureRandom.uuid) unless id.present?
+  end
+  
+end

data/lib/quandl/cassandra/models/data.rb

@@ -0,0 +1,18 @@
+class Quandl::Cassandra::Models::Data < Quandl::Data
+  
+  require_relative 'data/search'
+  
+  include Quandl::Cassandra::Models::Data::Search
+  
+  attr_accessor :dataset_id, :column_ids, :column_frequencies
+  
+  # start a new scope from this data
+  def scoped
+    s = self.class.scope.new
+    s.id(dataset_id)
+    s.column_ids(column_ids) if column_ids
+    s.column_frequencies(column_frequencies) if column_frequencies
+    s
+  end
+  
+end

data/lib/quandl/cassandra/models/data/search.rb

@@ -0,0 +1,105 @@
+module Quandl::Cassandra::Models::Data::Search
+  
+  extend ActiveSupport::Concern
+  
+  included do
+    
+    include ScopeComposer::Model
+  
+    def self.scope_names
+      scope.scope_names
+    end
+  
+    has_scope_composer
+  
+    delegate :where, to: :scope
+  
+    scope :dataset, ->(d){
+      id(d.id)
+      column_ids(d.column_ids)
+    }
+  
+    scope :row, :id, :limit, :offset, :column, :accuracy, :frequency, :count, :delete
+    
+    scope :column_frequencies, ->(*freqs){ where( column_frequencies: Array(freqs).flatten ) }
+    scope :column_ids, ->(*ids){ cids = Array(ids).flatten.compact; where( column_ids: cids ) if cids.present? }
+
+    scope :collapse,  ->(v){ where( collapse:   v.to_sym ) if Quandl::Operation::Collapse.valid_collapse?(v) }
+    scope :transform, ->(v){ where( transform:  v.to_sym ) if Quandl::Operation::Transform.valid_transformation?(v) }
+  
+    scope :order, ->(v){
+      order = ( v.to_sym == :asc ) ? :asc : :desc
+      where( order: order )
+    }
+  
+    scope :trim_start,  ->(date){ date = parse_date(date); where( trim_start: date ) if date }
+    scope :trim_end,    ->(date){ date = parse_date(date); where( trim_end:   date ) if date }
+  
+    scope_helper :find,       ->(id){ id(id).to_table }
+    scope_helper :to_table,   ->{ all }
+  
+    scope_helper :parse_date, ->( value ){ 
+      begin
+        date = Date.jd(value.to_i) if value.kind_of?(String) && value.numeric?
+        date = Date.jd(value) if value.is_a?(Integer)
+        date = Date.parse(value) if value.is_a?(String) && value =~ /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/
+        date = value if value.is_a?(Date)
+        date.jd
+      rescue
+        nil
+      end
+    }
+    
+    scope.class_eval do
+      
+      delegate :to_a, :==, :inspect, :first, :flatten, :[], :collect, :<=>, :each, :each_with_index, :to_date, :to_h, to: :all, allow_nil: true
+      
+      def delete_all(*args)
+        if attributes[:column_ids].present?
+          result = Quandl::Cassandra::Models::Column.where( id: attributes[:column_ids] ).delete_all
+          return result.nil? ? true : result
+        end
+        false
+      end
+      
+      def count(*args)
+        attributes[:count] = true
+        result = dataset? ? fetch.to_i : 0
+        attributes[:count] = false
+        result
+      end
+      
+      def all
+        @all ||= fetch
+      end
+      
+      def dataset?
+        self.id.present? || attributes[:column_ids].present?
+      end
+      
+      def fetched?
+        @all.present?
+      end
+      
+      def scoped
+        s = self.class.new
+        s.id( self.id ) if id.present?
+        s.column_ids( attributes[:column_ids] ) if attributes[:column_ids].present?
+        s.column_frequencies(attributes[:column_frequencies]) if attributes[:column_frequencies].present?
+        s
+      end
+      
+      protected
+      
+      def fetch
+        # without an id or columns there's nothing to be read
+        return Quandl::Cassandra::Models::Data.new unless dataset?
+        # otherwise read the data
+        Quandl::Cassandra::Models::Column.read( attributes.merge(scope_attributes) )
+      end
+      
+      
+    end
+    
+  end
+end