quandl_cassandra 0.0.1

data/lib/quandl/cassandra_models/column/write/group_data_by_column.rb

@@ -0,0 +1,42 @@
+class Quandl::Cassandra::Column::Write::GroupDataByColumn < Quandl::Cassandra::Column::Write
+  
+  # INPUTS
+  # { source: [ [1,2,3], [2,4,8], ... ], weekly: ... }
+  
+  # OUTPUTS
+  # { source: { UUID: [[1,2], [2,4]], UUID: [[1,3],[2,8]] }}
+  
+  def perform
+    self.frequency_column_data = {}
+    # for each { frequency: [ [12,3,4], ... ] }
+    frequency_data.each do |frequency, data|
+      # assign grouped data to frequency_column_data
+      self.frequency_column_data[frequency] = group_data_by_column(data)
+    end
+  end
+  
+  def group_data_by_column(data)
+    column_data = {}
+    # for each [ [date, val, val], ... ]
+    data.each do |row|
+      # extract date
+      date = row[0]
+      # for each [ val, val, ... ]
+      row[1..-1].each_with_index do |value, index|
+        # ensure array
+        column_data[ column_id(index) ] ||= []
+        # group each each [date, value] by column_id, excluding nil
+        column_data[ column_id(index) ] << [date, value] unless value.blank?
+      end
+    end
+    column_data
+  end
+  
+  def column_id(index)
+    # ensure column_ids is defined
+    self.column_ids ||= Quandl::Cassandra::Dataset.find_column_ids_by_id(id)
+    # ensure column_ids[index] is present
+    self.column_ids[index] ||= SecureRandom.uuid
+  end
+  
+end

data/lib/quandl/cassandra_models/column/write/group_data_by_frequency.rb

@@ -0,0 +1,24 @@
+class Quandl::Cassandra::Column::Write::GroupDataByFrequency < Quandl::Cassandra::Column::Write
+  
+  # INPUTS
+  # [ [1,2,3], [2,4,8], ... ]
+  
+  # OUTPUTS
+  # { source: [ [1,2,3], [2,4,8], ... ], weekly: ... }
+  
+  def perform
+    self.frequency = data.frequency
+    self.frequency_data = {}
+    # clone source data
+    self.data = data.clone
+    self.frequency_data[:source] = data.data_array.clone
+    # collapse and clone each frequency
+    Quandl::Operation::Collapse.collapses_greater_than(data.frequency).each do |freq|
+      # collapse the data to the required frequency
+      data.collapse(freq)
+      # clone the internal data array and add it to the grouping
+      self.frequency_data[freq] = data.data_array.clone
+    end
+  end
+  
+end

data/lib/quandl/cassandra_models/column/write/insert_columns.rb

@@ -0,0 +1,22 @@
+class Quandl::Cassandra::Column::Write::InsertColumns < Quandl::Cassandra::Column::Write
+  
+  # INPUTS
+  # { source: { UUID: [[1,2], [2,4]], UUID: [[1,3],[2,8]] }}
+  
+  def perform
+    return if column_ids.blank?
+    column_ids.each_with_index{|column_id, position| 
+      Quandl::Cassandra::Base.execute( datasets_statement( column_id, position ) ) 
+      Quandl::Cassandra::Base.execute( column_attributes_statement( column_id ) ) 
+    }
+  end
+  
+  def datasets_statement( column_id, position )
+    "INSERT INTO datasets (id, column_id, position) VALUES (#{id}, #{column_id}, #{position})"
+  end
+  
+  def column_attributes_statement( column_id )
+    "INSERT INTO column_attributes ( id, frequency ) VALUES ( #{column_id}, '#{frequency}' )"
+  end
+  
+end

data/lib/quandl/cassandra_models/column/write/insert_data.rb

@@ -0,0 +1,39 @@
+class Quandl::Cassandra::Column::Write::InsertData < Quandl::Cassandra::Column::Write
+  
+  def perform
+    insert_data_in_batches.collect(&:value)
+  end
+  
+  def insert_data_in_batches
+    futures = []
+    statements = []
+    frequency_column_data.each do |frequency, column_data|
+      column_data.each do |column_id, rows|
+        rows.each do |time_value|
+          # collect statements
+          statements << statement( column_id, frequency, time_value[0], time_value[1] )
+          # after 30 statements are collected, execute a batch insert
+          if statements.count >= Quandl::Cassandra.configuration.batch_size
+            # collect the futures
+            futures << execute_async_batch(statements)
+            # clear statements
+            statements = []
+          end
+        end
+      end
+    end
+    # execute any remaining statements
+    futures << execute_async_batch(statements) if statements.count > 0
+    futures
+  end
+  
+  def execute_async_batch(statements)
+    batch = %Q{BEGIN UNLOGGED BATCH\n#{statements.join("\n")}\nAPPLY BATCH;}
+    future = Quandl::Cassandra::Base.execute_async( batch )
+  end
+  
+  def statement( id, type, time, value )
+    "INSERT INTO columns (id, type, time, value) VALUES (#{id}, '#{type}', #{time}, #{value})"
+  end
+  
+end

data/lib/quandl/cassandra_models/column/write.rb

@@ -0,0 +1,22 @@
+class Quandl::Cassandra::Column::Write < Quandl::Strategy::Strategize
+  
+  # strategy attributes
+  define_attributes :id, :data, :frequency, :column_ids, :frequency_data, :frequency_column_data
+  
+  require_relative 'write/insert_data'
+  require_relative 'write/insert_columns'
+  require_relative 'write/group_data_by_column'
+  require_relative 'write/group_data_by_frequency'
+  
+  # execute strategy
+  def self.perform(attributes)
+    strategy = Quandl::Strategy.new( attributes ) do |c|
+      c.use Quandl::Cassandra::Column::Write::GroupDataByFrequency
+      c.use Quandl::Cassandra::Column::Write::GroupDataByColumn
+      c.use Quandl::Cassandra::Column::Write::InsertData
+      c.use Quandl::Cassandra::Column::Write::InsertColumns
+    end
+    strategy.perform
+  end
+  
+end

data/lib/quandl/cassandra_models/column.rb

@@ -0,0 +1,20 @@
+class Quandl::Cassandra::Column < Quandl::Cassandra::Base
+
+  table_name :columns
+
+  require_relative 'column/read'
+  require_relative 'column/write'
+  
+  class << self
+    
+    def read(*args)
+      Quandl::Cassandra::Column::Read.perform(*args)[:data]
+    end
+    
+    def write(*args)
+      Quandl::Cassandra::Column::Write.perform(*args)
+    end
+    
+  end
+  
+end

data/lib/quandl/cassandra_models/column_attribute.rb

@@ -0,0 +1,11 @@
+class Quandl::Cassandra::ColumnAttribute < Quandl::Cassandra::Base
+
+  table_name :column_attributes
+
+  after_initialize :default_attributes
+
+  def default_attributes
+    self.id = SecureRandom.uuid
+  end
+  
+end

data/lib/quandl/cassandra_models/data.rb

@@ -0,0 +1,52 @@
+class Quandl::Cassandra::Data
+  
+  include ScopeComposer::Model
+  
+  has_scope_composer
+  
+  delegate :where, to: :scope
+  
+  scope :dataset, ->(d){ id(d.id).column_ids(d.column_ids) }
+  
+  scope :row, :id, :limit, :offset, :column, :accuracy, :frequency
+  
+  scope :column_frequencies, ->(*freqs){ where( column_frequencies: Array(freqs).flatten ) }
+  scope :column_ids, ->(*ids){ where( column_ids: Array(ids).flatten ) }
+
+  scope :collapse,  ->(v){ where( collapse:   v.to_sym ) if Quandl::Operation::Collapse.valid_collapse?(v) }
+  scope :transform, ->(v){ where( transform:  v.to_sym ) if Quandl::Operation::Transform.valid_transformation?(v) }
+  
+  scope :order, ->(v){
+    order = ( v.to_sym == :asc ) ? :asc : :desc
+    where( order: order )
+  }
+  
+  scope :trim_start,  ->(date){ where( trim_start:  parse_date(date).jd ) }
+  scope :trim_end,    ->(date){ where( trim_end:    parse_date(date).jd ) }
+  
+  scope_helper :find,       ->(id){ id(id).to_table }
+  scope_helper :to_table,   ->{ all }
+  
+  scope_helper :parse_date, ->( date ){ 
+    begin
+      date = Date.jd(date.to_i) if date.kind_of?(String) && date.numeric?
+      date = Date.jd(date) if date.is_a?(Integer)
+      date = Date.parse(date) if date.is_a?(String) && date =~ /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/
+      date
+    rescue
+      nil
+    end
+  }
+  
+  scope.class_eval do
+    
+    delegate :inspect, :==, to: :all, allow_nil: true
+    Array.forwardable_methods.each{|mname| delegate(mname, to: :all, allow_nil: true ) unless self.respond_to?(mname) }
+    
+    def all
+      @all ||= Quandl::Cassandra::Column.read( attributes.merge(scope_attributes) )
+    end
+    
+  end
+
+end

data/lib/quandl/cassandra_models/dataset.rb

@@ -0,0 +1,83 @@
+class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
+
+  table_name :datasets
+  autosave_changes false
+  
+  define_attributes :id, :data, :column_ids
+  
+  before_save :save_columns, :save_data, :save_dataset_attribute
+  after_save :clear_attributes
+  
+  delegate :type, :updated_at, :created_at, :frequency, to: :dataset_attribute, allow_nil: true
+
+  def self.find_column_ids_by_id(id)
+    Dataset.where( id: id ).pluck(:column_id, :position).sort_by{|r| r[1] }.collect{|r| r[0] }
+  end
+
+  def column_attributes=(column_attrs)
+    column_attrs.each_with_index do |attrs, index|
+      self.columns[index] ||=  Quandl::Cassandra::Column.new
+      self.columns[index].assign_attributes(attrs)
+    end
+  end
+   
+  def columns
+    @columns ||= column_ids.collect{|cid| Quandl::Cassandra::ColumnAttribute.find(cid) }
+  end
+  
+  def column_ids
+    @column_ids ||= self.class.find_column_ids_by_id(id)
+  end
+  
+  def trim_start
+    @trim_start ||= data_scope.limit(1).order(:asc).try(:[], 0).try(:[], 0)
+  end
+  
+  def trim_end
+    @trim_end ||= data_scope.limit(1).order(:desc).try(:[], 0).try(:[], 0)
+  end
+  
+  def data
+    # data set?
+    return read_attribute(:data) if data?
+    # read data
+    @attributes[:data] ||= data_scope
+  end
+  
+  def data=(rows)
+    rows = Quandl::Data.new(rows) unless rows.is_a?(Quandl::Data)
+    write_attribute(:data, rows)
+  end
+  
+  def data_scope
+    Quandl::Cassandra::Data.dataset(self)
+  end
+  
+  def dataset_attribute
+    @dataset_attribute ||= Quandl::Cassandra::DatasetAttribute.find_or_build(id)
+  end
+  
+  protected
+  
+  def save_dataset_attribute
+    dataset_attribute.frequency = data.frequency.to_s
+    dataset_attribute.save
+  end
+  
+  def save_columns
+    columns.each(&:save)
+  end
+  
+  def save_data
+    Quandl::Cassandra::Column.write( id: id, data: data ) if data_changed?
+  end
+  
+  def clear_attributes
+    @trim_start = nil
+    @trim_end = nil
+    @columns = nil
+    @column_ids = nil
+    @attributes[:data] = nil
+  end
+  
+end

data/lib/quandl/cassandra_models/dataset_attribute.rb

@@ -0,0 +1,6 @@
+class Quandl::Cassandra::DatasetAttribute < Quandl::Cassandra::Base
+
+  table_name :dataset_attributes
+  define_attributes :id, :type, :updated_at, :created_at, :frequency
+  
+end

data/lib/quandl/cassandra_models/multiset.rb

@@ -0,0 +1,50 @@
+class Quandl::Cassandra::Multiset < Quandl::Cassandra::Dataset
+  
+  table_name :datasets
+  
+  define_attributes :datasets_columns
+  
+  class << self
+    
+    def with_columns(datasets_columns)
+      self.new( datasets_columns: datasets_columns )
+    end
+    
+  end
+  
+  def data_scope
+    Quandl::Cassandra::Data.collapse(:source).column_ids( column_ids ).column_frequencies( column_frequencies )
+  end
+  
+  def column_frequencies
+    @column_frequencies ||= columns.collect(&:frequency)
+  end
+  
+  def column_ids
+    @column_ids ||= column_ids_from_datasets_columns
+  end
+  
+  def column_ids_from_datasets_columns
+    ids = []
+    datasets_columns.split(',').each do |dataset_column|
+      dataset_id, column = dataset_column.split('.')
+      datasets[dataset_id] ||= Quandl::Cassandra::Dataset.find(dataset_id).column_ids
+      ids << datasets[dataset_id][ column.to_i - 1 ]
+    end
+    ids
+  end
+  
+  def datasets
+    @datasets ||= {}
+  end
+  
+  def frequency
+    data.frequency
+  end
+  
+  def dataset_attribute
+    # find or initialize dataset attribute object
+    @dataset_attribute ||= Quandl::Cassandra::DatasetAttribute.new
+  end
+  
+end

data/lib/quandl/strategy.rb

@@ -0,0 +1,59 @@
+class Quandl::Strategy
+  
+  attr_accessor :attributes
+  
+  def initialize(*args, &block)
+    setup(*args, &block)
+  end
+  
+  def setup(attributes, &block)
+    self.attributes = attributes
+    block.call(self) if block_given?
+  end
+  
+  def perform
+    classes.each do |klass|
+      strategy = klass.new(attributes)
+      self.attributes = strategy.attributes
+    end
+    attributes
+  end
+  
+  def use(klass)
+    self.classes << klass unless classes.include?(klass)
+  end
+  
+  def classes
+    @classes ||= []
+  end
+  
+  class Strategize
+      
+    attr_accessor :attributes
+    
+    def self.call(attrs)
+      self.new(attrs).attributes
+    end
+      
+    def initialize(attrs)
+      attrs = attrs.clone if attrs.respond_to?(:clone)
+      self.attributes = attrs
+      self.perform if respond_to?(:perform)
+    end
+    
+    def self.define_attributes(*names)
+      names.each do |name|
+        # getter
+        define_method(name) do
+          self.attributes[name.to_sym]
+        end
+        # setter
+        define_method("#{name}=") do |value|
+          self.attributes[name.to_sym] = value
+        end
+      end
+    end
+      
+  end
+  
+end