quandl_cassandra 0.0.1 → 0.1.0

data/lib/quandl/cassandra/base/attributes.rb

@@ -42,6 +42,7 @@ module Attributes
   end
   
   def initialize(*args)
+    @attributes = {}
     self.assign_attributes(args.extract_options!)
   end
   
@@ -63,12 +64,11 @@ module Attributes
     @attributes = value.symbolize_keys! if value.is_a?(Hash)
   end
   
-  def save
-    cycle_changes!
-  end
-  
   def reload
+    return false unless primary_key?
     clear_changes!
+    reload_attributes
+    true
   end
   
   protected

data/lib/quandl/cassandra/base/persistence.rb

@@ -23,21 +23,27 @@ module Quandl::Cassandra::Base::Persistence
     def new_from_query_result(result)
       return nil if result.blank?
       r = self.new
+      r.new_record = false
       r.attributes = result
       r
     end
     
   end
   
+  attr_accessor :new_record
+  
   def save!
     save
   end
   
   def save
+    return false if self.class.autosave_changes && !primary_key?
     run_callbacks :save do
       touch
       save_changes! if self.class.autosave_changes == true
-      clear_changes!
+      cycle_changes!
+      new_record = false
+      true
     end
   end
   
@@ -56,8 +62,25 @@ module Quandl::Cassandra::Base::Persistence
     self.class.execute("INSERT INTO #{self.class.table_name} (#{values.keys.join(', ')}) VALUES (#{values.values.join(', ')})")
   end
   
+  def new_record?
+    new_record
+  end
+  
+  def new_record
+    return @new_record if defined?(@new_record)
+    @new_record = true
+  end
+  
+  def new_record=(value)
+    @new_record = (value == true)
+  end
+  
   private
   
+  def reload_attributes
+    self.attributes = self.class.where(primary_key_attributes).limit(1).to_h.first
+  end
+  
   def save_changes!
     attrs = {}
     changes.each{|k,v| attrs[k] = v[1] }

data/lib/quandl/cassandra/base/sanitization.rb

@@ -20,12 +20,15 @@ module Quandl::Cassandra::Base::Sanitization
         # exclude attributes that are not present in columns
         next unless column_type(attribute)
         # sanitize value for cql
-        values[attribute] = sanitize_attribute(attribute, value)# unless value.nil?
+        values[attribute] = sanitize_attribute(attribute, value)
       end
       values
     end
   
     def sanitize_attribute(attribute, value)
+      # process each value
+      return value.collect{|val| column_type(attribute).sanitize_for_cql( val ) } if value.is_a?(Array)
+      # process value
       column_type(attribute).sanitize_for_cql( value )
     end
     

data/lib/quandl/cassandra/base/schema.rb

@@ -1,6 +1,14 @@
 module Quandl::Cassandra::Base::Schema
   
   extend ActiveSupport::Concern
+    
+  def primary_key?
+    !primary_key_attributes.values.include?(nil)
+  end
+    
+  def primary_key_attributes
+    self.class.primary_key.inject({}){|m,key| m[key] = self.send(key); m }
+  end
   
   module ClassMethods
     

data/lib/quandl/cassandra/base/scoping.rb

@@ -4,6 +4,10 @@ module Quandl::Cassandra::Base::Scoping
   
   module ClassMethods
     
+    def scope_names
+      scope.scope_names
+    end
+    
     def where(*args)
       scope.new.where(*args)
     end
@@ -16,10 +20,13 @@ module Quandl::Cassandra::Base::Scoping
   
     has_scope_composer
     
-    scope :select, :limit, :order, :count, :find, :first, :all, :pluck, :exists?
+    scope :select, :delete_all, :limit,
+      :order, :count, :find, :first, :all, :pluck, :exists?, :to_h
     
     scope.class_eval do
       
+      attr_accessor :statement_type
+      
       delegate :to_a, to: :execute
       
       delegate :primary_key, :table_name, :sanitize_attributes, to: :parent
@@ -41,18 +48,28 @@ module Quandl::Cassandra::Base::Scoping
         select(*columns).execute.present?
       end
       
+      def to_h
+        execute
+      end
+      
       def pluck(*args)
         r = select(*args).values
         r = r.flatten if args.count <= 1
         r
       end
       
+      def select=(*names)
+        self.statement_type = :select
+        write_scope_attribute(:select, names)
+      end
+      
       def values
         to_a.collect(&:values)
       end
       
       def find(*args)
         primary_key = parent.bind_primary_key(*args)
+        return nil if primary_key.blank?
         where(primary_key).first
       end
       
@@ -69,17 +86,44 @@ module Quandl::Cassandra::Base::Scoping
       end
       
       def to_cql
-        ["SELECT #{cql_select} FROM", table_name, cql_where, cql_order, cql_limit].compact.join(' ')
+        [cql_statement_type, table_name, cql_where, cql_order, cql_limit].compact.join(' ')
+      end
+      
+      def delete_all
+        self.statement_type = :delete
+        execute
+      end
+      
+      def insert
+        self.statement_type = :insert
+        self
       end
       
       protected
       
+      def cql_statement_type
+        case statement_type
+        when :insert then "INSERT INTO"
+        when :delete then "DELETE FROM" 
+        else
+          "SELECT #{cql_select} FROM"
+        end
+      end
+      
       def cql_where
         attrs = attributes.clone
         attrs = sanitize_attributes(attrs) if parent.table_exists?
-        attrs = attrs.collect{|k,v| "#{k} = #{v}" }.join(' AND ')
+        attrs = attrs.collect{|key, value| cql_where_attribute(key, value) }.join(' AND ')
         cql = attrs.present? ? "WHERE #{attrs}" : nil
       end
+      
+      def cql_where_attribute(key, value)
+        if value.is_a?(Array)
+          "#{key} IN (#{value.join(',')})"
+        else
+          "#{key} = #{value}"
+        end
+      end
   
       def cql_select
         columns = Array(select).join(',') if select.present?

data/lib/quandl/cassandra/types/boolean_type.rb

@@ -4,7 +4,7 @@ class Quandl::Cassandra::Types::BooleanType < Quandl::Cassandra::Types::Abstract
     return true if value == true || value.to_s =~ (/(true|t|yes|y|1)$/i)
     return false if value == false || value.blank? || value.to_s =~ (/(false|f|no|n|0)$/i)
   rescue
-    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{self.class}", value)
+    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{name} #{value}", value)
   end
   
 end

data/lib/quandl/cassandra/types/decimal_type.rb

@@ -3,7 +3,7 @@ class Quandl::Cassandra::Types::DecimalType < Quandl::Cassandra::Types::Abstract
   def self.sanitize(value)
     Float(value)
   rescue
-    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{self.class}", value)
+    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{name} #{value}", value)
   end
   
 end

data/lib/quandl/cassandra/types/double_type.rb

@@ -3,7 +3,7 @@ class Quandl::Cassandra::Types::DoubleType < Quandl::Cassandra::Types::AbstractT
   def self.sanitize(value)
     Float(value)
   rescue
-    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{self.class}", value)
+    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{name} #{value}", value)
   end
   
 end

data/lib/quandl/cassandra/types/float_type.rb

@@ -3,7 +3,7 @@ class Quandl::Cassandra::Types::FloatType < Quandl::Cassandra::Types::AbstractTy
   def self.sanitize(value)
     Float(value)
   rescue
-    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{self.class}", value)
+    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{self.name} #{value}", value)
   end
   
 end

data/lib/quandl/cassandra/types/integer_type.rb

@@ -3,7 +3,7 @@ class Quandl::Cassandra::Types::IntegerType < Quandl::Cassandra::Types::Abstract
   def self.sanitize(value)
     Integer(value)
   rescue
-    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{self.class}", value)
+    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{self.name} #{value}", value)
   end
   
 end

data/lib/quandl/cassandra/types/long_type.rb

@@ -3,7 +3,7 @@ class Quandl::Cassandra::Types::LongType < Quandl::Cassandra::Types::AbstractTyp
   def self.sanitize(value)
     Integer(value)
   rescue
-    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{self.class}", value)
+    raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{self.name} #{value}", value)
   end
   
 end

data/lib/quandl/cassandra/types/timestamp_type.rb

@@ -9,7 +9,7 @@ class Quandl::Cassandra::Types::TimestampType < Quandl::Cassandra::Types::Abstra
     return Time.at(value.get_time / 1000.0) if value.respond_to?(:get_time)
     Time.at(value.to_f) if value.present?
   rescue => error
-    raise Quandl::Cassandra::Error::CastException.new("Unable to convert to Timestamp #{value} #{error}", value)
+    raise Quandl::Cassandra::Error::CastException.new("Unable to convert to #{name} #{value}", value)
   end
   
 end

data/lib/quandl/cassandra/types/utf8_type.rb

@@ -7,7 +7,7 @@ class Quandl::Cassandra::Types::UTF8Type < Quandl::Cassandra::Types::AbstractTyp
   def self.sanitize(value)
     RUBY_VERSION >= "1.9" ? value.to_s.dup.force_encoding('UTF-8') : value.to_s.dup
   rescue => e
-    raise Quandl::Cassandra::Error::CastException.new("Unable to convert to UTF8", caller)
+    raise Quandl::Cassandra::Error::CastException.new("Unable to convert to #{name} #{value}", caller)
   end
   
 end

data/lib/quandl/cassandra/types/uuid_type.rb

@@ -4,7 +4,7 @@ module Quandl::Cassandra::Types
   
     def self.sanitize(value)
       value = value.to_s
-      raise Quandl::Cassandra::Error::CastException, "Unable to convert to UUID", caller unless value =~ regex_validator
+      raise Quandl::Cassandra::Error::CastException, "Unable to convert to #{name} #{value}", caller unless value =~ regex_validator
       value
     rescue => e
       raise 

data/lib/quandl/cassandra/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Cassandra
-  VERSION = '0.0.1'
+  VERSION = '0.1.0'
 end
 end

data/lib/quandl/cassandra_models/column/read/collapse.rb

@@ -22,11 +22,12 @@ class Quandl::Cassandra::Column::Read::Collapse < Quandl::Cassandra::Column::Rea
     # frequency is the given frequency
     freq = attributes[:frequency]
     # grab from database
-    freq = DatasetAttribute.where(id: id).pluck(:frequency).first if freq.blank? and id.present?
+    freq = Quandl::Cassandra::DatasetAttribute.where(id: id).pluck(:frequency).first if freq.blank? and id.present?
     # fallback to collapse
     freq = attributes[:collapse] if freq.blank?
     # if freq is source, daily is being requested
     freq = :daily if freq == :source
+    freq = freq.to_sym
     freq
   end
   

data/lib/quandl/cassandra_models/column/read/column.rb

@@ -2,9 +2,10 @@ class Quandl::Cassandra::Column::Read::Column < Quandl::Cassandra::Column::Read
   
   def perform
     # given an id, this is a dataset
-    attributes[:column_ids] ||= Quandl::Cassandra::Dataset.find(id).column_ids if id.present?
+    attributes[:column_ids] ||= Quandl::Cassandra::Dataset.find(id).try(:column_ids) if id.present?
+    attributes[:column_ids] = [] if attributes[:column_ids].blank?
     # given a column, this is requesting a specific column
-    attributes[:column_ids] = pluck_column_id if column.present?
+    attributes[:column_ids] = pluck_column_id if column.present? && attributes[:column_ids].present?
   end
   
   def pluck_column_id

data/lib/quandl/cassandra_models/column/read/{data_table.rb → data.rb}

@@ -1,13 +1,15 @@
-class Quandl::Cassandra::Column::Read::DataTable < Quandl::Cassandra::Column::Read
+class Quandl::Cassandra::Column::Read::Data < Quandl::Cassandra::Column::Read
   
   def perform
-    attributes[:data] = quandl_data( attributes[:data] )
+    attributes[:data] = quandl_data( attributes[:data] ) unless count?
   end
   
-  
   def quandl_data(data)
     # init
-    data = Quandl::Data.new( data )
+    data = Quandl::Cassandra::Data.new( data )
+    data.dataset_id = attributes[:id]
+    data.column_ids = attributes[:column_ids] unless attributes[:column].present?
+    data.column_frequencies = attributes[:column_frequencies] unless attributes[:column].present?
     # post process
     data = collapse(data)
     data = transform(data)

data/lib/quandl/cassandra_models/column/read/offset.rb

@@ -2,6 +2,7 @@ class Quandl::Cassandra::Column::Read::Offset < Quandl::Cassandra::Column::Read
   
   def perform
     # apply offset
+    return unless column_ids.present?
     apply_offset_with_transform
     apply_offset
   end
@@ -60,6 +61,9 @@ class Quandl::Cassandra::Column::Read::Offset < Quandl::Cassandra::Column::Read
   end
   
   def accuracy_with_limit
+    # # short circuit for now
+    # return 0
+    # revist this at a later date:
     awl = accuracy
     awl = awl + limit - 1 if limit && limit > 0
     awl

data/lib/quandl/cassandra_models/column/read/{query.rb → select_columns.rb}

@@ -1,8 +1,20 @@
-class Quandl::Cassandra::Column::Read::Query < Quandl::Cassandra::Column::Read
+class Quandl::Cassandra::Column::Read::SelectColumns < Quandl::Cassandra::Column::Read
   
   def perform
     # attrs to result hash
-    attributes[:data] = select_data
+    attributes[:data] = count? ? count_data : select_data
+  end
+  
+  def count_data
+    prepared = Quandl::Cassandra::Base.prepare( statement )
+    rows = []
+    attributes[:column_ids].each_with_index do | id, index |
+      # pluck column type from collapses
+      type = attributes[:column_collapses][index].to_s
+      # bind and execute query
+      rows << prepared.execute( id, type, :one )
+    end
+    rows.collect{|r| r.first['count'] }.max
   end
   
   def select_data
@@ -42,14 +54,19 @@ class Quandl::Cassandra::Column::Read::Query < Quandl::Cassandra::Column::Read
   end
   
   def statement
-    cql = "SELECT time,value FROM columns WHERE"
+    columns = count? ? "COUNT(*)" : "time,value"
+    cql = "SELECT #{columns} FROM columns WHERE"
     # cql += "ORDER"
     cql += " time >= #{attributes[:trim_start]} AND " if attributes[:trim_start]
     cql += " time <= #{attributes[:trim_end]} AND " if attributes[:trim_end]
     cql += " id = ? AND type = ?"
-    cql += " ORDER BY type #{attributes[:order]}" if attributes[:order]
+    cql += " ORDER BY type #{order}"
     cql += " LIMIT #{attributes[:limit]}" if attributes[:limit]
     cql
   end
   
+  def order
+    @order ||= attributes[:order] == :asc ? :asc : :desc
+  end
+  
 end

data/lib/quandl/cassandra_models/column/read.rb

@@ -2,9 +2,9 @@ class Quandl::Cassandra::Column::Read < Quandl::Strategy::Strategize
 
   require_relative 'read/collapse'
   require_relative 'read/column'
-  require_relative 'read/data_table'
+  require_relative 'read/data'
   require_relative 'read/offset'
-  require_relative 'read/query'
+  require_relative 'read/select_columns'
   require_relative 'read/row'
   require_relative 'read/transform'
   require_relative 'read/type'
@@ -19,10 +19,14 @@ class Quandl::Cassandra::Column::Read < Quandl::Strategy::Strategize
       c.use Quandl::Cassandra::Column::Read::Collapse
       c.use Quandl::Cassandra::Column::Read::Transform
       c.use Quandl::Cassandra::Column::Read::Offset
-      c.use Quandl::Cassandra::Column::Read::Query
-      c.use Quandl::Cassandra::Column::Read::DataTable
+      c.use Quandl::Cassandra::Column::Read::SelectColumns
+      c.use Quandl::Cassandra::Column::Read::Data
     end
     strategy.perform
   end
-
+  
+  def count?
+    attributes[:count] == true
+  end
+  
 end

data/lib/quandl/cassandra_models/column/write/insert_column_attributes.rb

@@ -0,0 +1,22 @@
+class Quandl::Cassandra::Column::Write::InsertColumnAttributes < Quandl::Cassandra::Column::Write
+  
+  # INPUTS
+  # { source: { UUID: [[1,2], [2,4]], UUID: [[1,3],[2,8]] }}
+  
+  def perform
+    return if column_ids.blank?
+    column_ids.each_with_index{|column_id, position| 
+      Quandl::Cassandra::Base.execute( datasets_statement( column_id, position ) ) 
+      Quandl::Cassandra::Base.execute( column_attributes_statement( column_id ) ) 
+    }
+  end
+  
+  def datasets_statement( column_id, position )
+    "INSERT INTO datasets (id, column_id, position) VALUES (#{id}, #{column_id}, #{position})"
+  end
+  
+  def column_attributes_statement( column_id )
+    "INSERT INTO column_attributes ( id, frequency ) VALUES ( #{column_id}, '#{frequency}' )"
+  end
+  
+end

data/lib/quandl/cassandra_models/column/write/insert_columns.rb

@@ -1,22 +1,39 @@
 class Quandl::Cassandra::Column::Write::InsertColumns < Quandl::Cassandra::Column::Write
   
-  # INPUTS
-  # { source: { UUID: [[1,2], [2,4]], UUID: [[1,3],[2,8]] }}
-  
   def perform
-    return if column_ids.blank?
-    column_ids.each_with_index{|column_id, position| 
-      Quandl::Cassandra::Base.execute( datasets_statement( column_id, position ) ) 
-      Quandl::Cassandra::Base.execute( column_attributes_statement( column_id ) ) 
-    }
+    insert_data_in_batches.collect(&:value)
+  end
+  
+  def insert_data_in_batches
+    futures = []
+    statements = []
+    frequency_column_data.each do |frequency, column_data|
+      column_data.each do |column_id, rows|
+        rows.each do |time_value|
+          # collect statements
+          statements << statement( column_id, frequency, time_value[0], time_value[1] )
+          # after 30 statements are collected, execute a batch insert
+          if statements.count >= Quandl::Cassandra.configuration.batch_size
+            # collect the futures
+            futures << execute_async_batch(statements)
+            # clear statements
+            statements = []
+          end
+        end
+      end
+    end
+    # execute any remaining statements
+    futures << execute_async_batch(statements) if statements.count > 0
+    futures
   end
   
-  def datasets_statement( column_id, position )
-    "INSERT INTO datasets (id, column_id, position) VALUES (#{id}, #{column_id}, #{position})"
+  def execute_async_batch(statements)
+    batch = %Q{BEGIN UNLOGGED BATCH\n#{statements.join("\n")}\nAPPLY BATCH;}
+    future = Quandl::Cassandra::Base.execute_async( batch )
   end
   
-  def column_attributes_statement( column_id )
-    "INSERT INTO column_attributes ( id, frequency ) VALUES ( #{column_id}, '#{frequency}' )"
+  def statement( id, type, time, value )
+    "INSERT INTO columns (id, type, time, value) VALUES (#{id}, '#{type}', #{time}, #{value})"
   end
   
 end

data/lib/quandl/cassandra_models/column/write.rb

@@ -3,8 +3,8 @@ class Quandl::Cassandra::Column::Write < Quandl::Strategy::Strategize
   # strategy attributes
   define_attributes :id, :data, :frequency, :column_ids, :frequency_data, :frequency_column_data
   
-  require_relative 'write/insert_data'
   require_relative 'write/insert_columns'
+  require_relative 'write/insert_column_attributes'
   require_relative 'write/group_data_by_column'
   require_relative 'write/group_data_by_frequency'
   
@@ -13,8 +13,8 @@ class Quandl::Cassandra::Column::Write < Quandl::Strategy::Strategize
     strategy = Quandl::Strategy.new( attributes ) do |c|
       c.use Quandl::Cassandra::Column::Write::GroupDataByFrequency
       c.use Quandl::Cassandra::Column::Write::GroupDataByColumn
-      c.use Quandl::Cassandra::Column::Write::InsertData
       c.use Quandl::Cassandra::Column::Write::InsertColumns
+      c.use Quandl::Cassandra::Column::Write::InsertColumnAttributes
     end
     strategy.perform
   end

data/lib/quandl/cassandra_models/data/search.rb

@@ -0,0 +1,104 @@
+module Quandl::Cassandra::Data::Search
+  
+  extend ActiveSupport::Concern
+  
+  included do
+    
+    include ScopeComposer::Model
+  
+    def self.scope_names
+      scope.scope_names
+    end
+  
+    has_scope_composer
+  
+    delegate :where, to: :scope
+  
+    scope :dataset, ->(d){
+      id(d.id)
+      column_ids(d.column_ids)
+    }
+  
+    scope :row, :id, :limit, :offset, :column, :accuracy, :frequency, :count, :delete
+    
+    scope :column_frequencies, ->(*freqs){ where( column_frequencies: Array(freqs).flatten ) }
+    scope :column_ids, ->(*ids){ cids = Array(ids).flatten.compact; where( column_ids: cids ) if cids.present? }
+
+    scope :collapse,  ->(v){ where( collapse:   v.to_sym ) if Quandl::Operation::Collapse.valid_collapse?(v) }
+    scope :transform, ->(v){ where( transform:  v.to_sym ) if Quandl::Operation::Transform.valid_transformation?(v) }
+  
+    scope :order, ->(v){
+      order = ( v.to_sym == :asc ) ? :asc : :desc
+      where( order: order )
+    }
+  
+    scope :trim_start,  ->(date){ date = parse_date(date); where( trim_start: date ) if date }
+    scope :trim_end,    ->(date){ date = parse_date(date); where( trim_end:   date ) if date }
+  
+    scope_helper :find,       ->(id){ id(id).to_table }
+    scope_helper :to_table,   ->{ all }
+  
+    scope_helper :parse_date, ->( value ){ 
+      begin
+        date = Date.jd(value.to_i) if value.kind_of?(String) && value.numeric?
+        date = Date.jd(value) if value.is_a?(Integer)
+        date = Date.parse(value) if value.is_a?(String) && value =~ /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/
+        date.jd
+      rescue
+        nil
+      end
+    }
+    
+    scope.class_eval do
+      
+      delegate :to_a, :==, :inspect, :first, :flatten, :[], :collect, :<=>, :each, :each_with_index, :to_date, :to_h, to: :all, allow_nil: true
+      
+      def delete_all(*args)
+        if attributes[:column_ids].present?
+          result = Quandl::Cassandra::Column.where( id: attributes[:column_ids] ).delete_all
+          return result.nil? ? true : result
+        end
+        false
+      end
+      
+      def count(*args)
+        attributes[:count] = true
+        result = dataset? ? fetch.to_i : 0
+        attributes[:count] = false
+        result
+      end
+      
+      def all
+        @all ||= fetch
+      end
+      
+      def dataset?
+        self.id.present? || attributes[:column_ids].present?
+      end
+      
+      def fetched?
+        @all.present?
+      end
+      
+      def scoped
+        s = self.class.new
+        s.id( self.id ) if id.present?
+        s.column_ids( attributes[:column_ids] ) if attributes[:column_ids].present?
+        s.column_frequencies(attributes[:column_frequencies]) if attributes[:column_frequencies].present?
+        s
+      end
+      
+      protected
+      
+      def fetch
+        # without an id or columns there's nothing to be read
+        return Quandl::Cassandra::Data.new unless dataset?
+        # otherwise read the data
+        Quandl::Cassandra::Column.read( attributes.merge(scope_attributes) )
+      end
+      
+      
+    end
+    
+  end
+end