quandl_cassandra 0.0.1

data/quandl_cassandra.gemspec

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "quandl/cassandra/version"
+
+Gem::Specification.new do |s|
+  s.name        = "quandl_cassandra"
+  s.version     = Quandl::Cassandra::VERSION
+  s.authors     = ["Blake Hilscher"]
+  s.email       = ["blake@hilscher.ca"]
+  s.homepage    = "http://blake.hilscher.ca/"
+  s.license     = "MIT"
+  s.summary     = "Quandl cassandra interface."
+  s.description = "Quandl cassandra interface. CQL."
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rake", "~> 10.0"
+  s.add_development_dependency "rspec", "~> 2.13"
+  s.add_development_dependency "factory_girl_rails"
+  s.add_development_dependency "fivemat", "~> 1.2"
+  s.add_development_dependency "pry"
+
+  s.add_runtime_dependency "activesupport", ">= 3.0.0"
+  s.add_runtime_dependency "activemodel", ">= 3.0.0"
+  
+  s.add_runtime_dependency "scope_composer", "~> 0.3"
+  s.add_runtime_dependency "quandl_data", "~> 1.0"
+  s.add_runtime_dependency "quandl_logger", "~> 0.1"
+  s.add_runtime_dependency "quandl_operation", "~> 0.1"
+  s.add_runtime_dependency 'cql-rb', '1.1.0.rc0'
+  
+end

data/spec/expectations/string.rb

@@ -0,0 +1,5 @@
+RSpec::Matchers.define :be_same_string_as do |expected|
+  match do |actual|
+    actual.to_s == expected.to_s
+  end
+end

data/spec/expectations/time.rb

@@ -0,0 +1,5 @@
+RSpec::Matchers.define :be_same_second_as do |expected|
+  match do |actual|
+    actual.to_i == expected.to_i
+  end
+end

data/spec/factories/dataset.rb

@@ -0,0 +1,8 @@
+FactoryGirl.define do
+
+  factory :dataset, class: Quandl::Cassandra::Dataset do
+    sequence(:id) { |n| "#{(Time.now.to_f * 1000).to_i}#{n}".to_i } 
+    data { Quandl::Fabricate::Data.rand( rows: 60, columns: 4, nils: false ).to_csv }
+  end
+
+end

data/spec/lib/quandl/cassandra/base/scoping_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+class TestModel < Quandl::Cassandra::Base; end
+
+describe Quandl::Cassandra::Base::Scoping do
+
+  subject{ TestModel }
+  
+  [:where, :select, :limit, :order].each do |name|
+    it{ should respond_to name }
+  end
+  
+  describe ".to_cql" do
+    
+    let(:scope) { TestModel.scope.new }
+    subject{ scope.to_cql }
+  
+    describe "#where" do
+      before(:each){ scope.where( id: 1 ) }
+      it{ should eq "SELECT * FROM test_models WHERE id = 1" }
+
+      describe "#limit" do
+        before(:each){ scope.limit( 10 ) }
+        it{ should eq "SELECT * FROM test_models WHERE id = 1 LIMIT 10" }
+
+        describe "#select" do
+          before(:each){ scope.select( :name, :value ) }
+          it{ should eq "SELECT name,value FROM test_models WHERE id = 1 LIMIT 10" }
+        
+          describe "#order" do
+            before(:each){ scope.order("name DESC") }
+            it{ should eq "SELECT name,value FROM test_models WHERE id = 1 ORDER BY name DESC LIMIT 10" }
+          end
+        
+        end
+      end
+    end
+  end
+  
+end

data/spec/lib/quandl/cassandra_models/column/write/group_data_by_frequency_spec.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Cassandra::Column::Write::GroupDataByFrequency do
+
+  let(:data){ Quandl::Fabricate::Data.rand( columns: 4, rows: 12, nils: false, frequency: :weekly ) }
+  let(:strategy){ Quandl::Cassandra::Column::Write::GroupDataByFrequency.new({ data: data }) }
+  
+  describe "#frequency_data" do
+    subject{ strategy.frequency_data }
+    its(:length){ should eq 4 }
+    
+    it "should have source" do
+      subject[:source].count.should eq 12
+    end
+    it "should have monthly" do
+      subject[:monthly].count.should <= 4
+    end
+    it "should have quarterly" do
+      subject[:quarterly].count.should <= 2
+    end
+    it "should have annual" do
+      subject[:annual].count.should eq 1
+    end
+    
+  end
+  
+end

data/spec/lib/quandl/cassandra_models/column/write_spec.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Cassandra::Column::Write do
+
+  let(:id){ rand(10000*10000) + 10000*10000 }
+  let(:data){ Quandl::Fabricate::Data.rand( columns: 4, rows: 12, nils: false, frequency: :weekly ) }
+  
+  it "should not alter the original data" do
+    source_data = data.to_a.collect{|r| r.clone }
+    r = Quandl::Cassandra::Column.write( id: id, data: data )
+    data.should eq source_data
+  end
+  
+end

data/spec/lib/quandl/cassandra_models/column_attribute_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe Quandl::Cassandra::ColumnAttribute do
+  
+  let(:id){ rand(10000*10000) + 10000*10000 }
+  let(:data){ Quandl::Fabricate::Data.rand( rows: 10, columns: 2 ) }
+  let(:dataset){ Quandl::Cassandra::Dataset.create( id: id, data: data ) }
+  
+  let(:column){ Quandl::Cassandra::ColumnAttribute.find(dataset.column_ids.first) }
+  
+  subject{ column }
+  
+  its(:id){ should eq dataset.column_ids.first }
+  its(:frequency){ should eq 'daily' }
+  
+end

data/spec/lib/quandl/cassandra_models/column_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe Quandl::Cassandra::Column do
+  
+  let(:id){ rand(10000*10000) + 10000*10000 }
+  let(:data){ Quandl::Fabricate::Data.rand( rows: 10, columns: 2 ) }
+  let(:dataset){ Quandl::Cassandra::Dataset.create( id: id, data: data ) }
+  
+  describe ".read" do
+    before(:each){ Quandl::Cassandra::Column.write( id: id, data: data ); sleep(0.2) }
+    
+    subject{ Quandl::Cassandra::Column.read( id: id ) }
+    its(:count){ should eq 10 }
+    it{ should eq data }
+  end
+  
+end

data/spec/lib/quandl/cassandra_models/data_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+describe Quandl::Cassandra::Data do
+
+  let(:id){ rand(10000*10000) + 10000*10000 }
+  let(:data){ Quandl::Fabricate::Data.rand(rows: 10, columns: 2, nils: false) }
+  let(:dataset){ Quandl::Cassandra::Dataset.create( id: id, data: data ) }
+  subject{ Quandl::Cassandra::Data }
+  
+  [:row, :id, :limit, :offset, :column, :accuracy, :frequency, :column_ids,
+    :collapse, :transform, :order, :trim_start, :trim_end ].each do |name|
+    it{ should respond_to name }
+  end
+  
+  describe "#collapse" do
+    it "should become annual" do
+      scope = subject.collapse(:annual)
+      scope.attributes[:collapse].should eq :annual
+    end
+    it "should become annual given string" do
+      scope = subject.collapse('annual')
+      scope.attributes[:collapse].should eq :annual
+    end
+    it "should become monthly" do
+      scope = subject.collapse(:monthly)
+      scope.attributes[:collapse].should eq :monthly
+    end
+    it "should become weekly" do
+      scope = subject.collapse('weekly')
+      scope.attributes[:collapse].should eq :weekly
+    end
+  end
+  
+end

data/spec/lib/quandl/cassandra_models/dataset/collapse_spec.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Cassandra::Dataset do
+  
+  let(:dataset){ create(:dataset) }
+  subject { dataset }
+  
+  it "should have columns" do
+    subject.columns.count.should eq 4
+  end
+  
+  describe "#data" do
+    
+    subject{ dataset.data }
+    
+    its(:class){ should eq Quandl::Cassandra::Data::ScopeScope }
+    
+    it "should have data" do
+      subject.count.should eq 60
+    end
+  
+    it "should collapse to daily" do
+      subject.collapse(:daily).count.should eq 60
+    end
+  
+    it "should collapse to weekly" do
+      subject.collapse(:weekly).count.should be < 12
+    end
+  
+    it "should collapse to monthly" do
+      subject.collapse(:monthly).count.should be < 4
+    end
+
+    it "should collapse to annual" do
+      subject.collapse(:annual).count.should eq 1
+    end
+  
+  end
+
+end

data/spec/lib/quandl/cassandra_models/dataset/column_spec.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Cassandra::Dataset do
+  
+  context "column" do
+    
+    subject { create(:dataset) }
+    
+    it "should return first column" do
+      subject.data.column(1)[0][1].should eq subject.data_scope.to_table[0][1]
+    end
+    
+    it "should return second column" do
+      # binding.pry
+      subject.data.column(2)[0][1].should eq subject.data_scope.to_table[0][2]
+    end
+    
+    it "should return third column" do
+      subject.data.column(2)[0][1].should eq subject.data_scope.to_table[0][2]
+    end
+    
+  end
+  
+end

data/spec/lib/quandl/cassandra_models/dataset/persistence_spec.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Cassandra::Dataset do
+  
+  let(:data){ Quandl::Fabricate::Data.rand( rows: 4, columns: 4, nils: false ) }
+  let(:dataset){ create( :dataset, data: data ) }
+  
+  subject{ dataset }
+  
+  its(:dataset_attribute){ should be_a DatasetAttribute }
+  its(:trim_start){ should eq Dataset.find(dataset.id).data[-1][0] }
+  its(:trim_end){ should eq Dataset.find(dataset.id).data[0][0] }
+  its(:updated_at){ should_not be_nil }
+  
+  context "after save" do
+    before(:each){ 
+      @previously_updated_at = subject.updated_at
+      subject.save 
+    }
+    its(:updated_at){ should_not eq @previously_updated_at }
+  end
+  
+end

data/spec/lib/quandl/cassandra_models/dataset/row_spec.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Cassandra::Dataset do
+  
+  let(:data){ Quandl::Fabricate::Data.rand( rows: 48, columns: 2, frequency: :monthly, nils: false ).to_csv }
+  let(:dataset){ create( :dataset, data: data ) }
+  subject { dataset }
+  
+  it "should be monthly" do
+    subject.frequency.should eq 'monthly'
+  end
+    
+  it "should pluck the second row" do
+    subject.data_scope.row(1).to_a.should eq [subject.data_scope[1]]
+  end
+    
+  it "should collapse and pluck the second year" do
+    subject.data_scope.collapse(:annual).row(2).to_a.should eq [subject.data_scope.collapse(:annual)[2]]
+  end
+    
+  it "should collapse and pluck the second year with a transformation" do
+    subject.data_scope.collapse(:annual).transform(:rdiff).row(2).to_a.should eq [subject.data_scope.collapse(:annual).transform(:rdiff)[2]]
+  end
+  
+end

data/spec/lib/quandl/cassandra_models/dataset/transform_spec.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Dataset do
+  
+  let(:data){ Quandl::Data.new([ [1002, 10], [1001, 20], [1000, 30] ]) }
+  let(:dataset){ build(:dataset) }
+
+  it "should cumul data" do
+    dataset.data = data
+    dataset.save!
+    cumul_data = Dataset.find(dataset.id).data.transform(:cumul).to_table.data_array
+    cumul_data.should eq [ [1002, 60.0], [1001, 50.0], [1000, 30.0] ]
+  end
+  
+end

data/spec/lib/quandl/cassandra_models/dataset/trim_spec.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Cassandra::Dataset do
+  
+  describe ".trim_start,.trim_end" do
+    { weekly: 7, monthly: 30, quarterly: 92, annual: 365 }.each do | collapse_to, occurences |
+      context "when collapsed to #{collapse_to}" do
+
+        subject(:dataset){ create(:dataset, data: Quandl::Fabricate::Data.rand( rows: occurences * 3, columns: 1, nils: false ).to_csv ) }
+        
+        it "should include trim_start" do
+          # raw data
+          source_data = subject.data_scope.collapse(collapse_to).to_table
+          trim_start = source_data[-1][0]
+          # trim and check
+          trim_data = subject.data.trim_start( trim_start ).collapse(collapse_to).to_table
+          trim_data.last[0].should eq trim_start
+        end
+        it "should include trim_end" do
+          # raw data
+          source_data = subject.data_scope.collapse(collapse_to).to_table
+          trim_end = source_data[1][0]
+          # trim and check
+          trim_data = subject.data.trim_end( trim_end ).collapse(collapse_to).to_table
+          trim_data.first[0].should eq trim_end
+        end
+        it "should include trim_start and trim_end" do
+          # raw data
+          source_data = subject.data_scope.collapse(collapse_to).to_table
+          trim_end = source_data[1][0]
+          trim_start = source_data[-1][0]
+          # trim and check
+          trim_data = subject.data.trim_start( trim_start ).trim_end( trim_end ).collapse(collapse_to).to_table
+          trim_data.first[0].should eq trim_end
+          trim_data.last[0].should eq trim_start
+        end
+        
+        [:diff, :rdiff, :cumul].each do |transformed_to|
+          context "when transformed to #{transformed_to}" do
+            it "should include trim_start" do
+              # raw data
+              source_data = subject.data_scope.collapse(collapse_to).transform(transformed_to).to_table
+              trim_start = source_data[-1][0]
+              # trim and check
+              trim_data = subject.data.trim_start( trim_start ).transform(transformed_to).collapse(collapse_to).to_table
+              trim_data.last[0].should eq trim_start
+            end
+            it "should include trim_end" do
+              # raw data
+              source_data = subject.data_scope.collapse(collapse_to).transform(transformed_to).to_table
+              trim_end = source_data[-1][0]
+              # trim and check
+              trim_data = subject.data.trim_end( trim_end ).transform(transformed_to).collapse(collapse_to).to_table
+              trim_data.first[0].should eq trim_end
+            end
+            it "should include trim_start and trim_end" do
+              source_data = subject.data_scope.collapse(collapse_to).transform(transformed_to).to_table
+              trim_end = source_data[1][0]
+              trim_start = source_data[-1][0]
+              # trim and check
+              trim_data = subject.data.trim_start( trim_start ).trim_end( trim_end ).transform(transformed_to).collapse(collapse_to).to_table
+              trim_data.first[0].should eq trim_end
+              trim_data.last[0].should eq trim_start
+            end
+          end # each transform
+
+        end
+        
+      end # each collapse
+    end
+  end
+  
+end

data/spec/lib/quandl/cassandra_models/dataset/update_spec.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Dataset do
+
+  context "update data with newer data" do
+    
+    subject{ create(:dataset, data: Quandl::Fabricate::Data.rand( rows: 730, columns: 2, nils: false ).to_csv ) }
+    
+    it "should update the collapse data" do
+      old_row = subject.data_scope[0]
+      old_row_month = subject.data_scope.collapse(:monthly)[0]
+      # update
+      dataset = Dataset.find(subject.id)
+      # advance data dates by 60 days
+      new_data = subject.data.collect{|r|
+        date = r[0] + 60
+        values = r[1..-1].collect{ rand(9102841).to_f / 1000 }
+        [date, values].flatten
+      }
+      # assign new data
+      dataset.data = new_data.to_a.collect{|r| r.to_csv }.join
+      dataset.save!
+      
+      new_row = dataset.data_scope[0]
+      new_row_month = dataset.data_scope.collapse(:monthly)[0]
+      
+      new_row[0].should_not eq new_row_month[0]
+      new_row[1].should eq new_row_month[1]
+      
+      old_row[0].should_not eq new_row[0]
+      old_row_month[1].should_not eq new_row_month[1]
+    end
+    
+  end
+  
+end