quandl_cassandra 0.3.1 → 1.0.0

data/lib/quandl/cassandra_models/column/read/row.rb

@@ -1,20 +0,0 @@
-class Quandl::Cassandra::Column::Read::Row < Quandl::Cassandra::Column::Read
-
-  def perform
-    return unless attributes[:row].present?
-    # ensure integer
-    row = attributes[:row]
-    # negative value needs inversion
-    if row < 0
-      attributes[:order]   = :asc
-      attributes[:offset]  = (row * -1) - 1
-      attributes[:limit]   = 1
-    # postive value is an offset of current observation
-    else
-      attributes[:order]   = :desc
-      attributes[:offset]  = row
-      attributes[:limit]   = 1
-    end
-  end
-  
-end

data/lib/quandl/cassandra_models/column/read/select_columns.rb

@@ -1,60 +0,0 @@
-class Quandl::Cassandra::Column::Read::SelectColumns < Quandl::Cassandra::Column::Read
-  
-  def perform
-    # attrs to result hash
-    attributes[:data] = count? ? count_data : select_data
-  end
-  
-  def count_data
-    prepared = Quandl::Cassandra::Base.prepare( statement )
-    rows = []
-    attributes[:column_ids].each_with_index do | id, index |
-      # pluck column type from collapses
-      type = attributes[:column_collapses][index].to_s
-      # bind and execute query
-      rows << prepared.execute( id, type, :one )
-    end
-    rows.collect{|r| r.first['count'] }.max
-  end
-  
-  def select_data
-    t1 = Time.now
-    # fire off the queries
-    prepared = Quandl::Cassandra::Base.prepare( statement )
-    data = {}
-    futures = []
-    attributes[:column_ids].each_with_index do | id, index |
-      # pluck column type from collapses
-      type = attributes[:column_collapses][index].to_s
-      # bind and execute query
-      futures << prepared.async.execute( id, type, Quandl::Cassandra::Base.consistency )
-    end
-    # collect the results
-    futures.each_with_index do |future, index|
-      # collect result 
-      future.value.each do |row|
-        data[row['time']] ||= Array.new( attributes[:column_ids].count )
-        data[row['time']][index] ||= row['value']
-      end
-    end
-    Quandl::Logger.debug("(#{t1.elapsed_ms}) #{self.class.name}.select_data")
-    data
-  end
-  
-  def statement
-    columns = count? ? "COUNT(*)" : "time,value"
-    cql = "SELECT #{columns} FROM columns WHERE"
-    # cql += "ORDER"
-    cql += " time >= #{attributes[:trim_start]} AND " if attributes[:trim_start]
-    cql += " time <= #{attributes[:trim_end]} AND " if attributes[:trim_end]
-    cql += " id = ? AND type = ?"
-    cql += " ORDER BY type #{order}"
-    cql += " LIMIT #{attributes[:limit]}" if attributes[:limit]
-    cql
-  end
-  
-  def order
-    @order ||= attributes[:order] == :asc ? :asc : :desc
-  end
-  
-end

data/lib/quandl/cassandra_models/column/read/transform.rb

@@ -1,53 +0,0 @@
-class Quandl::Cassandra::Column::Read::Transform < Quandl::Cassandra::Column::Read
-  
-  def perform
-    return unless attributes[:transform].present?
-    # apply
-    apply_rdiff
-    apply_cumul
-  end
-  
-  def apply_cumul
-    # rdiff_from needs the data from the current to offset
-    return unless transform?( :cumul ) && limit.present?    
-    # data table needs to handle the offset since cumul requires all data
-    attributes[:data_table] = Quandl::Cassandra::Column::Read::Offset.call(attributes)
-    # the query should not limit or offset the data
-    attributes[:row] = nil
-    attributes[:limit] = nil
-    attributes[:offset] = nil
-  end
-  
-  def apply_rdiff
-    return unless transform? :rdiff, :diff
-    # limit requires limit
-    attributes[:limit] = attributes[:limit] + 1 if attributes[:limit]
-    # trims should be increased by one
-    attributes[:trim_start] = trim_start.occurrences_of_frequency_ago( 1, attributes[:collapse] ).jd if trim_start.present? && order == :desc
-    attributes[:trim_end] = trim_end.occurrences_of_frequency_ahead( 1, attributes[:collapse] ).jd if trim_end.present? && order == :asc
-  end
-  
-  def transform?(*keys)
-    keys.each do |key|
-      return true if key == attributes[:transform]
-    end
-    false
-  end
-  
-  def trim_end
-    @trim_end ||= attributes[:trim_end].present? ? Date.jd( attributes[:trim_end] ) : nil
-  end
-  
-  def trim_start
-    @trim_start ||= attributes[:trim_start].present? ? Date.jd( attributes[:trim_start] ) : nil
-  end
-  
-  def limit
-    attributes[:limit]
-  end
-  
-  def order
-    @order ||= attributes[:order] == :asc ? :asc : :desc
-  end
-  
-end

data/lib/quandl/cassandra_models/column/read/type.rb

@@ -1,25 +0,0 @@
-class Quandl::Cassandra::Column::Read::Type < Quandl::Cassandra::Column::Read
-  
-  def perform
-    # enforce types
-    attributes.each do |key, value|
-      attributes[key] = enforce_type(key, value)
-    end
-    # retain pristine copy of attrs
-    attributes[:pristine] ||= attributes.clone
-  end
-  
-  def enforce_type(key, value)
-    case key
-    when :limit, :column, :trim_start, :trim_end, :offset, :accuracy, :row
-      return value.try(:to_i)
-    when :collapse, :transform, :frequency
-      return value.try(:to_sym)
-    when :order
-      return value.try(:to_sym) == :asc ? :asc : :desc
-    else
-      value
-    end
-  end
-  
-end

data/lib/quandl/cassandra_models/column/write.rb

@@ -1,22 +0,0 @@
-class Quandl::Cassandra::Column::Write < Quandl::Strategy::Strategize
-  
-  # strategy attributes
-  define_attributes :id, :data, :frequency, :column_ids, :frequency_data, :frequency_column_data, :statement_values
-  
-  require_relative 'write/insert_columns'
-  require_relative 'write/insert_column_attributes'
-  require_relative 'write/group_data_by_column'
-  require_relative 'write/group_data_by_frequency'
-  
-  # execute strategy
-  def self.perform(attributes)
-    strategy = Quandl::Strategy.new( attributes ) do |c|
-      c.use Quandl::Cassandra::Column::Write::GroupDataByFrequency
-      c.use Quandl::Cassandra::Column::Write::GroupDataByColumn
-      c.use Quandl::Cassandra::Column::Write::InsertColumns
-      c.use Quandl::Cassandra::Column::Write::InsertColumnAttributes
-    end
-    strategy.perform
-  end
-  
-end

data/lib/quandl/cassandra_models/column/write/group_data_by_column.rb

@@ -1,37 +0,0 @@
-class Quandl::Cassandra::Column::Write::GroupDataByColumn < Quandl::Cassandra::Column::Write
-  
-  # INPUTS
-  # { source: [ [1,2,3], [2,4,8], ... ], weekly: ... }
-  
-  # OUTPUTS
-  # { source: { UUID: [[1,2], [2,4]], UUID: [[1,3],[2,8]] }}
-  
-  def perform
-    group_by_statement_values
-  end
-  
-  def group_by_statement_values
-    # for each [ [date, val, val], ... ]
-    self.statement_values = []
-    frequency_data.each do |frequency, rows|
-      frequency = frequency.to_s
-      rows.each do |row|
-        # extract date
-        date = row[0]
-        # for each [ val, val, ... ]
-        row[1..-1].each_with_index do |value, index|
-          # ensure array
-          self.statement_values << [ column_id(index), frequency, date, value] unless value.blank?
-        end
-      end
-    end
-  end
-  
-  def column_id(index)
-    # ensure column_ids is defined
-    self.column_ids ||= Quandl::Cassandra::Dataset.find_column_ids_by_id(id)
-    # ensure column_ids[index] is present
-    self.column_ids[index] ||= Cql::Uuid.new(SecureRandom.uuid)
-  end
-  
-end

data/lib/quandl/cassandra_models/column/write/group_data_by_frequency.rb

@@ -1,24 +0,0 @@
-class Quandl::Cassandra::Column::Write::GroupDataByFrequency < Quandl::Cassandra::Column::Write
-  
-  # INPUTS
-  # [ [1,2,3], [2,4,8], ... ]
-  
-  # OUTPUTS
-  # { source: [ [1,2,3], [2,4,8], ... ], weekly: ... }
-  
-  def perform
-    self.frequency = data.frequency
-    self.frequency_data = {}
-    # clone source data
-    self.data = data.clone
-    self.frequency_data[:source] = data.data_array.clone
-    # collapse and clone each frequency
-    Quandl::Operation::Collapse.collapses_greater_than(data.frequency).each do |freq|
-      # collapse the data to the required frequency
-      data.collapse(freq)
-      # clone the internal data array and add it to the grouping
-      self.frequency_data[freq] = data.data_array.clone
-    end
-  end
-  
-end

data/lib/quandl/cassandra_models/column/write/insert_column_attributes.rb

@@ -1,20 +0,0 @@
-class Quandl::Cassandra::Column::Write::InsertColumnAttributes < Quandl::Cassandra::Column::Write
-  
-  # INPUTS
-  # { source: { UUID: [[1,2], [2,4]], UUID: [[1,3],[2,8]] }}
-  
-  def perform
-    return if column_ids.blank?
-    # format data for batch insertion
-    rows_values = []
-    column_ids.each_with_index do |column_id, position|
-      rows_values << [id, column_id, position, frequency]
-    end
-    # insert data
-    Quandl::Cassandra::Batch.insert(rows_values) do |id, column_id, position, frequency|
-      %Q{INSERT INTO datasets (id, column_id, position) VALUES (#{id}, #{column_id}, #{position})
-      INSERT INTO column_attributes ( id, frequency ) VALUES ( #{column_id}, '#{frequency}' )}
-    end
-  end
-  
-end

data/lib/quandl/cassandra_models/column/write/insert_columns.rb

@@ -1,9 +0,0 @@
-class Quandl::Cassandra::Column::Write::InsertColumns < Quandl::Cassandra::Column::Write
-  
-  def perform
-    Quandl::Cassandra::Batch.insert(statement_values) do |id, type, time, value|
-      "INSERT INTO columns (id, type, time, value) VALUES (#{id}, '#{type}', #{time}, #{value})"
-    end
-  end
-  
-end

data/lib/quandl/cassandra_models/column_attribute.rb

@@ -1,11 +0,0 @@
-class Quandl::Cassandra::ColumnAttribute < Quandl::Cassandra::Base
-
-  table_name :column_attributes
-
-  after_initialize :default_attributes
-
-  def default_attributes
-    self.id = SecureRandom.uuid
-  end
-  
-end

data/lib/quandl/cassandra_models/data.rb

@@ -1,18 +0,0 @@
-class Quandl::Cassandra::Data < Quandl::Data
-  
-  require_relative 'data/search'
-  
-  include Quandl::Cassandra::Data::Search
-  
-  attr_accessor :dataset_id, :column_ids, :column_frequencies
-  
-  # start a new scope from this data
-  def scoped
-    s = self.class.scope.new
-    s.id(dataset_id)
-    s.column_ids(column_ids) if column_ids
-    s.column_frequencies(column_frequencies) if column_frequencies
-    s
-  end
-  
-end

data/lib/quandl/cassandra_models/data/search.rb

@@ -1,104 +0,0 @@
-module Quandl::Cassandra::Data::Search
-  
-  extend ActiveSupport::Concern
-  
-  included do
-    
-    include ScopeComposer::Model
-  
-    def self.scope_names
-      scope.scope_names
-    end
-  
-    has_scope_composer
-  
-    delegate :where, to: :scope
-  
-    scope :dataset, ->(d){
-      id(d.id)
-      column_ids(d.column_ids)
-    }
-  
-    scope :row, :id, :limit, :offset, :column, :accuracy, :frequency, :count, :delete
-    
-    scope :column_frequencies, ->(*freqs){ where( column_frequencies: Array(freqs).flatten ) }
-    scope :column_ids, ->(*ids){ cids = Array(ids).flatten.compact; where( column_ids: cids ) if cids.present? }
-
-    scope :collapse,  ->(v){ where( collapse:   v.to_sym ) if Quandl::Operation::Collapse.valid_collapse?(v) }
-    scope :transform, ->(v){ where( transform:  v.to_sym ) if Quandl::Operation::Transform.valid_transformation?(v) }
-  
-    scope :order, ->(v){
-      order = ( v.to_sym == :asc ) ? :asc : :desc
-      where( order: order )
-    }
-  
-    scope :trim_start,  ->(date){ date = parse_date(date); where( trim_start: date ) if date }
-    scope :trim_end,    ->(date){ date = parse_date(date); where( trim_end:   date ) if date }
-  
-    scope_helper :find,       ->(id){ id(id).to_table }
-    scope_helper :to_table,   ->{ all }
-  
-    scope_helper :parse_date, ->( value ){ 
-      begin
-        date = Date.jd(value.to_i) if value.kind_of?(String) && value.numeric?
-        date = Date.jd(value) if value.is_a?(Integer)
-        date = Date.parse(value) if value.is_a?(String) && value =~ /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/
-        date.jd
-      rescue
-        nil
-      end
-    }
-    
-    scope.class_eval do
-      
-      delegate :to_a, :==, :inspect, :first, :flatten, :[], :collect, :<=>, :each, :each_with_index, :to_date, :to_h, to: :all, allow_nil: true
-      
-      def delete_all(*args)
-        if attributes[:column_ids].present?
-          result = Quandl::Cassandra::Column.where( id: attributes[:column_ids] ).delete_all
-          return result.nil? ? true : result
-        end
-        false
-      end
-      
-      def count(*args)
-        attributes[:count] = true
-        result = dataset? ? fetch.to_i : 0
-        attributes[:count] = false
-        result
-      end
-      
-      def all
-        @all ||= fetch
-      end
-      
-      def dataset?
-        self.id.present? || attributes[:column_ids].present?
-      end
-      
-      def fetched?
-        @all.present?
-      end
-      
-      def scoped
-        s = self.class.new
-        s.id( self.id ) if id.present?
-        s.column_ids( attributes[:column_ids] ) if attributes[:column_ids].present?
-        s.column_frequencies(attributes[:column_frequencies]) if attributes[:column_frequencies].present?
-        s
-      end
-      
-      protected
-      
-      def fetch
-        # without an id or columns there's nothing to be read
-        return Quandl::Cassandra::Data.new unless dataset?
-        # otherwise read the data
-        Quandl::Cassandra::Column.read( attributes.merge(scope_attributes) )
-      end
-      
-      
-    end
-    
-  end
-end

data/lib/quandl/cassandra_models/dataset.rb

@@ -1,72 +0,0 @@
-class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
-  
-  require_relative 'dataset/columns'
-  
-  include Quandl::Cassandra::Dataset::Columns
-  
-  table_name :datasets
-  autosave_changes false
-  
-  define_attributes :id, :data
-  
-  before_save :save_data, :save_dataset_attribute
-  after_save :clear_attributes!
-  
-  delegate :type, :updated_at, :created_at, :frequency, to: :dataset_attribute, allow_nil: true
-  
-  def trim_start
-    @trim_start ||= Date.jd( data.scoped.limit(1).order(:asc)[0][0] )
-  rescue
-    nil
-  end
-  
-  def trim_end
-    @trim_end ||= Date.jd( data.scoped.limit(1).order(:desc)[0][0] )
-  rescue
-    nil
-  end
-  
-  def data
-    # data set?
-    return read_attribute(:data) if data?
-    # read data
-    @attributes[:data] ||= data_scope
-  end
-  
-  def data=(rows)
-    rows = Quandl::Data.new(rows) unless rows.is_a?(Quandl::Data)
-    data_will_change!
-    @attributes[:data] = rows
-  end
-  
-  def data_scope
-    Quandl::Cassandra::Data.dataset(self)
-  end
-  
-  def dataset_attribute
-    @dataset_attribute ||= Quandl::Cassandra::DatasetAttribute.find_or_build(id)
-  end
-  
-  def reload
-    clear_attributes!
-  end
-  
-  protected
-  
-  def save_dataset_attribute
-    dataset_attribute.frequency = data.frequency.to_s
-    dataset_attribute.save
-  end
-  
-  def save_data
-    Quandl::Cassandra::Column.write( id: id, data: data ) if data_changed?
-  end
-  
-  def clear_attributes!
-    super if defined?(super)
-    @trim_start = nil
-    @trim_end = nil
-    @attributes = { id: id }
-  end
-  
-end