quandl_cassandra_models 0.3.6

data/lib/quandl/cassandra/models/dataset.rb

@@ -0,0 +1,87 @@
+class Quandl::Cassandra::Models::Dataset < Quandl::Cassandra::Base
+  
+  require_relative 'dataset/columns'
+  
+  table_name :datasets
+  autosave_changes false
+  
+  define_attributes :id, :data
+  
+  after_initialize :repair_frequency
+  
+  before_save :save_data, :save_dataset_attribute
+  
+  after_save :clear_attributes!
+  
+  delegate :type, :updated_at, :created_at, :frequency, to: :dataset_attribute, allow_nil: true
+  
+  include Quandl::Cassandra::Models::Dataset::Columns
+  
+  def trim_start
+    @trim_start ||= Date.jd( data.scoped.limit(1).order(:asc)[0][0] )
+  rescue
+    nil
+  end
+  
+  def trim_end
+    @trim_end ||= Date.jd( data.scoped.limit(1).order(:desc)[0][0] )
+  rescue
+    nil
+  end
+  
+  def data
+    # data set?
+    return read_attribute(:data) if data?
+    # read data
+    @attributes[:data] ||= data_scope
+  end
+  
+  def data=(rows)
+    rows = Quandl::Data.new(rows) unless rows.is_a?(Quandl::Data)
+    rows = rows.to_jd
+    data_will_change!
+    @attributes[:data] = rows
+  end
+  
+  def data_scope
+    Quandl::Cassandra::Models::Data.dataset(self)
+  end
+  
+  def dataset_attribute
+    @dataset_attribute ||= Quandl::Cassandra::Models::DatasetAttribute.find_or_build(id)
+  end
+  
+  def reload
+    clear_attributes!
+  end
+  
+  protected
+  
+  def save_dataset_attribute
+    dataset_attribute.save if dataset_attribute.changed?
+  end
+  
+  def save_data
+    Quandl::Cassandra::Models::Column.write( id: id, data: data ) if data_changed?
+  end
+  
+  def clear_attributes!
+    super if defined?(super)
+    @trim_start = nil
+    @trim_end = nil
+    @dataset_attribute = nil
+    @attributes = { id: id }
+  end
+  
+  def repair_frequency
+    return if self.new_record?
+    if frequency.blank?
+      # repair dataset attribute
+      dataset_attribute.frequency = data_scope.limit(100).to_table.frequency.to_s
+      dataset_attribute.save
+    end
+  rescue => e
+    Quandl::Logger.error("#{self.class.name}.id #{id} #{e}")
+  end
+  
+end

data/lib/quandl/cassandra/models/dataset/columns.rb

@@ -0,0 +1,63 @@
+module Quandl::Cassandra::Models::Dataset::Columns
+  
+  extend ActiveSupport::Concern
+  
+  included do
+    # define_attributes :column_ids
+    before_save :save_columns
+
+  end
+  
+  module ClassMethods
+
+    def find_column_ids_by_id(id)
+      Quandl::Cassandra::Models::Dataset.where( id: id ).pluck(:column_id, :position).sort_by{|r| r[1] }.collect{|r| r[0] }
+    end
+    
+  end
+  
+  def column_attributes=(column_attrs)
+    column_attrs.each_with_index do |attrs, index|
+      self.columns[index] ||=  Quandl::Cassandra::Models::Column.new
+      self.columns[index].assign_attributes(attrs)
+    end
+  end
+
+  def column_units
+    @column_units ||= columns.collect(&:unit)
+  end
+
+  def column_names
+    @column_names ||= columns.collect(&:name)
+  end
+
+  def columns
+    return @columns if @columns
+    # nothing to do without column ids
+    return [] if column_ids.compact.blank?
+    # find columns
+    columns = Quandl::Cassandra::Models::ColumnAttribute.where( id: column_ids ).all
+    # build column where column_id was not found
+    @columns = column_ids.collect{|cid| columns.detect{|c| c.id.to_s == cid.to_s } || Quandl::Cassandra::Models::ColumnAttribute.new( id: cid ) }
+  end
+
+  def column_ids
+    @column_ids ||= id.blank? ? [] : self.class.find_column_ids_by_id(id)
+  end
+
+
+  protected
+  
+  def clear_attributes!
+    @columns = nil
+    @column_ids = nil
+  end
+  
+  
+  private
+  
+  def save_columns
+    columns.each{|c| c.save if c.changed? || c.new_record? }
+  end
+
+end

data/lib/quandl/cassandra/models/dataset_attribute.rb

@@ -0,0 +1,6 @@
+class Quandl::Cassandra::Models::DatasetAttribute < Quandl::Cassandra::Base
+
+  table_name :dataset_attributes
+  define_attributes :id, :type, :updated_at, :created_at, :frequency
+  
+end

data/lib/quandl/cassandra/models/multiset.rb

@@ -0,0 +1,55 @@
+class Quandl::Cassandra::Models::Multiset < Quandl::Cassandra::Models::Dataset
+  
+  table_name :datasets
+  
+  define_attributes :datasets_columns
+  
+  class << self
+    
+    def with_columns(datasets_columns)
+      self.new( datasets_columns: datasets_columns )
+    end
+    
+  end
+  
+  def data_scope
+    Quandl::Cassandra::Models::Data.collapse(:source).column_ids( column_ids ).column_frequencies( column_frequencies )
+  end
+  
+  def column_frequencies
+    @column_frequencies ||= columns.collect(&:frequency)
+  end
+  
+  def column_ids
+    @column_ids ||= column_ids_from_datasets_columns
+  end
+  
+  def column_ids_from_datasets_columns
+    ids = []
+    datasets_columns.to_s.split(',').each do |dataset_column|
+      dataset_id, column = dataset_column.split('.')
+      datasets[dataset_id] ||= Quandl::Cassandra::Models::Dataset.find(dataset_id).try(:column_ids)
+      ids << datasets[dataset_id][ column.to_i - 1 ] if datasets[dataset_id].is_a?(Array)
+    end
+    ids
+  end
+  
+  def datasets
+    @datasets ||= {}
+  end
+  
+  def frequency
+    return @frequency if @frequency
+    uniq_freqs = column_frequencies.compact.uniq
+    Quandl::Operation::Collapse.valid_collapses.each do |collapse|
+      @frequency = collapse.to_s if uniq_freqs.include?(collapse.to_s)
+    end
+    @frequency
+  end
+  
+  def dataset_attribute
+    # find or initialize dataset attribute object
+    @dataset_attribute ||= Quandl::Cassandra::Models::DatasetAttribute.new
+  end
+  
+end

data/lib/quandl/cassandra/models/version.rb

@@ -0,0 +1,7 @@
+module Quandl
+module Cassandra
+module Models
+  VERSION = '0.3.6'
+end
+end
+end

data/migrations/20131105204200_create_datasets.rb

@@ -0,0 +1,18 @@
+class CreateDatasets < CassandraMigrations::Migration
+  
+  def up
+    cql = "CREATE TABLE datasets (
+      id double,
+      column_id uuid,
+      position int,
+      PRIMARY KEY (id, column_id)
+    )
+    WITH COMPACT STORAGE;"
+    CassandraMigrations::Cassandra.execute(cql)
+  end
+
+  def down
+    drop_table :datasets
+  end
+  
+end

data/migrations/20131105204201_create_columns.rb

@@ -0,0 +1,18 @@
+class CreateColumns < CassandraMigrations::Migration
+  
+  def up
+    cql = "CREATE TABLE columns (
+    id uuid,
+    type text,
+    time int,
+    value double,
+    PRIMARY KEY (id, type, time)
+    ) WITH COMPACT STORAGE;"
+    CassandraMigrations::Cassandra.execute(cql)
+  end
+
+  def down
+    drop_table :columns
+  end
+  
+end

data/migrations/20131105204202_create_dataset_attributes.rb

@@ -0,0 +1,17 @@
+class CreateDatasetAttributes < CassandraMigrations::Migration
+  
+  def up
+    cql = "CREATE TABLE dataset_attributes (
+    id double PRIMARY KEY,
+    created_at timestamp,
+    frequency text,
+    type text,
+    updated_at timestamp) WITH caching='ALL';"
+    CassandraMigrations::Cassandra.execute(cql)
+  end
+
+  def down
+    drop_table :dataset_attributes
+  end
+  
+end

data/migrations/20131105204203_create_column_attributes.rb

@@ -0,0 +1,17 @@
+class CreateColumnAttributes < CassandraMigrations::Migration
+  
+  def up
+    cql = "CREATE TABLE column_attributes (
+    id uuid PRIMARY KEY,
+    frequency text,
+    name text,
+    units text
+    ) WITH caching='ALL';"
+    CassandraMigrations::Cassandra.execute(cql)
+  end
+
+  def down
+    drop_table :column_attributes
+  end
+  
+end

data/quandl_cassandra_models.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "quandl/cassandra/models/version"
+
+Gem::Specification.new do |s|
+  s.name        = "quandl_cassandra_models"
+  s.version     = Quandl::Cassandra::Models::VERSION
+  s.authors     = ["Blake Hilscher"]
+  s.email       = ["blake@hilscher.ca"]
+  s.homepage    = "http://blake.hilscher.ca/"
+  s.license     = "MIT"
+  s.summary     = "Quandl cassandra interface."
+  s.description = "Quandl cassandra interface. CQL."
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rake", "~> 10.0"
+  s.add_development_dependency "rspec", "~> 2.13"
+  s.add_development_dependency "factory_girl_rails"
+  s.add_development_dependency "fivemat", "~> 1.2"
+  s.add_development_dependency "pry"
+
+  s.add_runtime_dependency "quandl_cassandra", "~> 1.1"
+  
+end

data/spec/expectations/string.rb

@@ -0,0 +1,5 @@
+RSpec::Matchers.define :be_same_string_as do |expected|
+  match do |actual|
+    actual.to_s == expected.to_s
+  end
+end

data/spec/expectations/time.rb

@@ -0,0 +1,5 @@
+RSpec::Matchers.define :be_same_second_as do |expected|
+  match do |actual|
+    actual.to_i == expected.to_i
+  end
+end

data/spec/factories/dataset.rb

@@ -0,0 +1,8 @@
+FactoryGirl.define do
+
+  factory :dataset, class: Quandl::Cassandra::Models::Dataset do
+    sequence(:id) { |n| "#{(Time.now.to_f * 1000).to_i}#{n}".to_i } 
+    data { Quandl::Fabricate::Data.rand( rows: 60, columns: 4, nils: false ).to_csv }
+  end
+
+end

data/spec/lib/quandl/cassandra/models/column/read_spec.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Cassandra::Models::Column::Write do
+
+  let(:id){ rand(10000*10000) + 10000*10000 }
+  let(:data){ Quandl::Fabricate::Data.rand( columns: 1, rows: 15, nils: false, frequency: :annual ) }
+  before(:each){ Quandl::Cassandra::Models::Column.write( id: id, data: data ) }
+
+  attrs = {}
+  {
+    transform:  :rdiff,
+    collapse:   :annual,
+    row:        0, 
+    column:     1,
+  }.each do |key, value|
+    attrs[key] = value
+    it "should read #{attrs.to_query}" do
+      attrs[:id] = id
+      attrs[:column_ids] = Quandl::Cassandra::Dataset.find(id).column_ids
+      Quandl::Cassandra::Models::Column.read(attrs).should be_present
+    end
+    
+  end
+
+  
+end

data/spec/lib/quandl/cassandra/models/column/write/group_data_by_frequency_spec.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Cassandra::Models::Column::Write::GroupDataByFrequency do
+
+  let(:data){ Quandl::Fabricate::Data.rand( columns: 4, rows: 12, nils: false, frequency: :weekly ) }
+  let(:strategy){ Quandl::Cassandra::Models::Column::Write::GroupDataByFrequency.new({ data: data }) }
+  
+  describe "#frequency_data" do
+    subject{ strategy.frequency_data }
+    its(:length){ should eq 4 }
+    
+    it "should have source" do
+      subject[:source].count.should eq 12
+    end
+    it "should have monthly" do
+      subject[:monthly].count.should <= 4
+    end
+    it "should have quarterly" do
+      subject[:quarterly].count.should <= 2
+    end
+    it "should have annual" do
+      subject[:annual].count.should eq 1
+    end
+    
+  end
+  
+end

data/spec/lib/quandl/cassandra/models/column/write_spec.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Cassandra::Models::Column::Write do
+
+  let(:id){ rand(10000*10000) + 10000*10000 }
+  let(:data){ Quandl::Fabricate::Data.rand( columns: 4, rows: 12, nils: false, frequency: :weekly ) }
+  
+  it "should not alter the original data" do
+    source_data = data.to_a.collect{|r| r.clone }
+    r = Quandl::Cassandra::Models::Column.write( id: id, data: data )
+    data.should eq source_data
+  end
+  
+  context "given tiny data array" do
+    let(:data){ Quandl::Fabricate::Data.rand( columns: 1, rows: 2, nils: false ) }
+    before(:each){ Quandl::Cassandra::Models::Column.write( id: id, data: data ) }
+    it "should have written the data" do
+      Quandl::Cassandra::Models::Column.read( id: id ).should eq data
+    end
+  end
+  
+end

data/spec/lib/quandl/cassandra/models/column_attribute_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe Quandl::Cassandra::Models::ColumnAttribute do
+  
+  let(:id){ rand(10000*10000) + 10000*10000 }
+  let(:data){ Quandl::Fabricate::Data.rand( rows: 10, columns: 2 ) }
+  let(:dataset){ Quandl::Cassandra::Models::Dataset.create( id: id, data: data ) }
+  
+  let(:column){ Quandl::Cassandra::Models::ColumnAttribute.find(dataset.column_ids.first) }
+  
+  subject{ column }
+  
+  its(:id){ should eq dataset.column_ids.first }
+  its(:frequency){ should eq 'daily' }
+  
+end