quandl_cassandra 0.3.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/UPGRADE.md +6 -0
- data/lib/quandl/cassandra.rb +0 -7
- data/lib/quandl/cassandra/version.rb +1 -1
- data/spec/lib/quandl/cassandra/base/scoping_spec.rb +10 -10
- metadata +1 -61
- data/lib/quandl/cassandra_models/column.rb +0 -20
- data/lib/quandl/cassandra_models/column/read.rb +0 -32
- data/lib/quandl/cassandra_models/column/read/collapse.rb +0 -65
- data/lib/quandl/cassandra_models/column/read/column.rb +0 -19
- data/lib/quandl/cassandra_models/column/read/data.rb +0 -59
- data/lib/quandl/cassandra_models/column/read/offset.rb +0 -118
- data/lib/quandl/cassandra_models/column/read/row.rb +0 -20
- data/lib/quandl/cassandra_models/column/read/select_columns.rb +0 -60
- data/lib/quandl/cassandra_models/column/read/transform.rb +0 -53
- data/lib/quandl/cassandra_models/column/read/type.rb +0 -25
- data/lib/quandl/cassandra_models/column/write.rb +0 -22
- data/lib/quandl/cassandra_models/column/write/group_data_by_column.rb +0 -37
- data/lib/quandl/cassandra_models/column/write/group_data_by_frequency.rb +0 -24
- data/lib/quandl/cassandra_models/column/write/insert_column_attributes.rb +0 -20
- data/lib/quandl/cassandra_models/column/write/insert_columns.rb +0 -9
- data/lib/quandl/cassandra_models/column_attribute.rb +0 -11
- data/lib/quandl/cassandra_models/data.rb +0 -18
- data/lib/quandl/cassandra_models/data/search.rb +0 -104
- data/lib/quandl/cassandra_models/dataset.rb +0 -72
- data/lib/quandl/cassandra_models/dataset/columns.rb +0 -61
- data/lib/quandl/cassandra_models/dataset_attribute.rb +0 -6
- data/lib/quandl/cassandra_models/multiset.rb +0 -50
- data/spec/factories/dataset.rb +0 -8
- data/spec/lib/quandl/cassandra_models/column/write/group_data_by_frequency_spec.rb +0 -28
- data/spec/lib/quandl/cassandra_models/column/write_spec.rb +0 -23
- data/spec/lib/quandl/cassandra_models/column_attribute_spec.rb +0 -16
- data/spec/lib/quandl/cassandra_models/column_spec.rb +0 -17
- data/spec/lib/quandl/cassandra_models/data_spec.rb +0 -84
- data/spec/lib/quandl/cassandra_models/dataset/collapse_spec.rb +0 -44
- data/spec/lib/quandl/cassandra_models/dataset/column_spec.rb +0 -24
- data/spec/lib/quandl/cassandra_models/dataset/persistence_spec.rb +0 -24
- data/spec/lib/quandl/cassandra_models/dataset/row_spec.rb +0 -26
- data/spec/lib/quandl/cassandra_models/dataset/transform_spec.rb +0 -16
- data/spec/lib/quandl/cassandra_models/dataset/trim_spec.rb +0 -74
- data/spec/lib/quandl/cassandra_models/dataset/update_spec.rb +0 -37
- data/spec/lib/quandl/cassandra_models/dataset_attribute_spec.rb +0 -18
- data/spec/lib/quandl/cassandra_models/dataset_spec.rb +0 -111
- data/spec/lib/quandl/cassandra_models/multiset/collapse_spec.rb +0 -122
- data/spec/lib/quandl/cassandra_models/multiset/columns_spec.rb +0 -57
- data/spec/lib/quandl/cassandra_models/multiset/data_spec.rb +0 -25
- data/spec/lib/quandl/cassandra_models/multiset/transform_spec.rb +0 -69
@@ -1,61 +0,0 @@
|
|
1
|
-
module Quandl::Cassandra::Dataset::Columns
|
2
|
-
|
3
|
-
extend ActiveSupport::Concern
|
4
|
-
|
5
|
-
included do
|
6
|
-
# define_attributes :column_ids
|
7
|
-
before_save :save_columns
|
8
|
-
|
9
|
-
end
|
10
|
-
|
11
|
-
module ClassMethods
|
12
|
-
|
13
|
-
def find_column_ids_by_id(id)
|
14
|
-
Quandl::Cassandra::Dataset.where( id: id ).pluck(:column_id, :position).sort_by{|r| r[1] }.collect{|r| r[0] }
|
15
|
-
end
|
16
|
-
|
17
|
-
end
|
18
|
-
|
19
|
-
def column_attributes=(column_attrs)
|
20
|
-
column_attrs.each_with_index do |attrs, index|
|
21
|
-
self.columns[index] ||= Quandl::Cassandra::Column.new
|
22
|
-
self.columns[index].assign_attributes(attrs)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
def column_units
|
27
|
-
@column_units ||= columns.collect(&:unit)
|
28
|
-
end
|
29
|
-
|
30
|
-
def column_names
|
31
|
-
@column_names ||= columns.collect(&:name)
|
32
|
-
end
|
33
|
-
|
34
|
-
def columns
|
35
|
-
return @columns if @columns
|
36
|
-
# find columns
|
37
|
-
columns = Quandl::Cassandra::ColumnAttribute.where( id: column_ids ).all
|
38
|
-
# build column where column_id was not found
|
39
|
-
@columns = column_ids.collect{|cid| columns.detect{|c| c.id.to_s == cid.to_s } || Quandl::Cassandra::ColumnAttribute.new( id: cid ) }
|
40
|
-
end
|
41
|
-
|
42
|
-
def column_ids
|
43
|
-
@column_ids ||= id.blank? ? [] : self.class.find_column_ids_by_id(id)
|
44
|
-
end
|
45
|
-
|
46
|
-
|
47
|
-
protected
|
48
|
-
|
49
|
-
def clear_attributes!
|
50
|
-
@columns = nil
|
51
|
-
@column_ids = nil
|
52
|
-
end
|
53
|
-
|
54
|
-
|
55
|
-
private
|
56
|
-
|
57
|
-
def save_columns
|
58
|
-
columns.each(&:save)
|
59
|
-
end
|
60
|
-
|
61
|
-
end
|
@@ -1,50 +0,0 @@
|
|
1
|
-
class Quandl::Cassandra::Multiset < Quandl::Cassandra::Dataset
|
2
|
-
|
3
|
-
table_name :datasets
|
4
|
-
|
5
|
-
define_attributes :datasets_columns
|
6
|
-
|
7
|
-
class << self
|
8
|
-
|
9
|
-
def with_columns(datasets_columns)
|
10
|
-
self.new( datasets_columns: datasets_columns )
|
11
|
-
end
|
12
|
-
|
13
|
-
end
|
14
|
-
|
15
|
-
def data_scope
|
16
|
-
Quandl::Cassandra::Data.collapse(:source).column_ids( column_ids ).column_frequencies( column_frequencies )
|
17
|
-
end
|
18
|
-
|
19
|
-
def column_frequencies
|
20
|
-
@column_frequencies ||= columns.collect(&:frequency)
|
21
|
-
end
|
22
|
-
|
23
|
-
def column_ids
|
24
|
-
@column_ids ||= column_ids_from_datasets_columns
|
25
|
-
end
|
26
|
-
|
27
|
-
def column_ids_from_datasets_columns
|
28
|
-
ids = []
|
29
|
-
datasets_columns.to_s.split(',').each do |dataset_column|
|
30
|
-
dataset_id, column = dataset_column.split('.')
|
31
|
-
datasets[dataset_id] ||= Quandl::Cassandra::Dataset.find(dataset_id).try(:column_ids)
|
32
|
-
ids << datasets[dataset_id][ column.to_i - 1 ] if datasets[dataset_id].is_a?(Array)
|
33
|
-
end
|
34
|
-
ids
|
35
|
-
end
|
36
|
-
|
37
|
-
def datasets
|
38
|
-
@datasets ||= {}
|
39
|
-
end
|
40
|
-
|
41
|
-
def frequency
|
42
|
-
data.frequency
|
43
|
-
end
|
44
|
-
|
45
|
-
def dataset_attribute
|
46
|
-
# find or initialize dataset attribute object
|
47
|
-
@dataset_attribute ||= Quandl::Cassandra::DatasetAttribute.new
|
48
|
-
end
|
49
|
-
|
50
|
-
end
|
data/spec/factories/dataset.rb
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require 'spec_helper'
|
3
|
-
|
4
|
-
describe Quandl::Cassandra::Column::Write::GroupDataByFrequency do
|
5
|
-
|
6
|
-
let(:data){ Quandl::Fabricate::Data.rand( columns: 4, rows: 12, nils: false, frequency: :weekly ) }
|
7
|
-
let(:strategy){ Quandl::Cassandra::Column::Write::GroupDataByFrequency.new({ data: data }) }
|
8
|
-
|
9
|
-
describe "#frequency_data" do
|
10
|
-
subject{ strategy.frequency_data }
|
11
|
-
its(:length){ should eq 4 }
|
12
|
-
|
13
|
-
it "should have source" do
|
14
|
-
subject[:source].count.should eq 12
|
15
|
-
end
|
16
|
-
it "should have monthly" do
|
17
|
-
subject[:monthly].count.should <= 4
|
18
|
-
end
|
19
|
-
it "should have quarterly" do
|
20
|
-
subject[:quarterly].count.should <= 2
|
21
|
-
end
|
22
|
-
it "should have annual" do
|
23
|
-
subject[:annual].count.should eq 1
|
24
|
-
end
|
25
|
-
|
26
|
-
end
|
27
|
-
|
28
|
-
end
|
@@ -1,23 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require 'spec_helper'
|
3
|
-
|
4
|
-
describe Quandl::Cassandra::Column::Write do
|
5
|
-
|
6
|
-
let(:id){ rand(10000*10000) + 10000*10000 }
|
7
|
-
let(:data){ Quandl::Fabricate::Data.rand( columns: 4, rows: 12, nils: false, frequency: :weekly ) }
|
8
|
-
|
9
|
-
it "should not alter the original data" do
|
10
|
-
source_data = data.to_a.collect{|r| r.clone }
|
11
|
-
r = Quandl::Cassandra::Column.write( id: id, data: data )
|
12
|
-
data.should eq source_data
|
13
|
-
end
|
14
|
-
|
15
|
-
context "given tiny data array" do
|
16
|
-
let(:data){ Quandl::Fabricate::Data.rand( columns: 1, rows: 2, nils: false ) }
|
17
|
-
before(:each){ Quandl::Cassandra::Column.write( id: id, data: data ) }
|
18
|
-
it "should have written the data" do
|
19
|
-
Quandl::Cassandra::Column.read( id: id ).should eq data
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
end
|
@@ -1,16 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Quandl::Cassandra::ColumnAttribute do
|
4
|
-
|
5
|
-
let(:id){ rand(10000*10000) + 10000*10000 }
|
6
|
-
let(:data){ Quandl::Fabricate::Data.rand( rows: 10, columns: 2 ) }
|
7
|
-
let(:dataset){ Quandl::Cassandra::Dataset.create( id: id, data: data ) }
|
8
|
-
|
9
|
-
let(:column){ Quandl::Cassandra::ColumnAttribute.find(dataset.column_ids.first) }
|
10
|
-
|
11
|
-
subject{ column }
|
12
|
-
|
13
|
-
its(:id){ should eq dataset.column_ids.first }
|
14
|
-
its(:frequency){ should eq 'daily' }
|
15
|
-
|
16
|
-
end
|
@@ -1,17 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Quandl::Cassandra::Column do
|
4
|
-
|
5
|
-
let(:id){ rand(10000*10000) + 10000*10000 }
|
6
|
-
let(:data){ Quandl::Fabricate::Data.rand( rows: 10, columns: 2, nils: false ) }
|
7
|
-
let(:dataset){ Quandl::Cassandra::Dataset.create( id: id, data: data ) }
|
8
|
-
|
9
|
-
describe ".read" do
|
10
|
-
before(:each){ Quandl::Cassandra::Column.write( id: id, data: data ); sleep(0.2) }
|
11
|
-
|
12
|
-
subject{ Quandl::Cassandra::Column.read( id: id ) }
|
13
|
-
its(:count){ should eq 10 }
|
14
|
-
it{ should eq data }
|
15
|
-
end
|
16
|
-
|
17
|
-
end
|
@@ -1,84 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Quandl::Cassandra::Data do
|
4
|
-
|
5
|
-
let(:id){ rand(10000*10000) + 10000*10000 }
|
6
|
-
let(:data){ Quandl::Fabricate::Data.rand(rows: 10, columns: 2, nils: false) }
|
7
|
-
let(:dataset){ Quandl::Cassandra::Dataset.create( id: id, data: data ) }
|
8
|
-
subject{ Quandl::Cassandra::Data }
|
9
|
-
|
10
|
-
[:row, :id, :limit, :offset, :column, :accuracy, :frequency, :column_ids,
|
11
|
-
:collapse, :transform, :order, :trim_start, :trim_end ].each do |name|
|
12
|
-
it{ should respond_to name }
|
13
|
-
end
|
14
|
-
|
15
|
-
describe "#collapse" do
|
16
|
-
it "should become annual" do
|
17
|
-
scope = subject.collapse(:annual)
|
18
|
-
scope.attributes[:collapse].should eq :annual
|
19
|
-
end
|
20
|
-
it "should become annual given string" do
|
21
|
-
scope = subject.collapse('annual')
|
22
|
-
scope.attributes[:collapse].should eq :annual
|
23
|
-
end
|
24
|
-
it "should become monthly" do
|
25
|
-
scope = subject.collapse(:monthly)
|
26
|
-
scope.attributes[:collapse].should eq :monthly
|
27
|
-
end
|
28
|
-
it "should become weekly" do
|
29
|
-
scope = subject.collapse('weekly')
|
30
|
-
scope.attributes[:collapse].should eq :weekly
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
describe "#trim_start" do
|
35
|
-
|
36
|
-
subject{ dataset.data }
|
37
|
-
|
38
|
-
context "given invalid date" do
|
39
|
-
|
40
|
-
it "rejects string" do
|
41
|
-
subject.trim_start('invalid').attributes[:trim_start].should be_nil
|
42
|
-
end
|
43
|
-
|
44
|
-
end
|
45
|
-
|
46
|
-
context "given valid date" do
|
47
|
-
|
48
|
-
it "accepts string" do
|
49
|
-
subject.trim_start('1980-01-01').attributes[:trim_start].should eq Date.parse('1980-01-01').jd
|
50
|
-
end
|
51
|
-
|
52
|
-
it "accepts julian date" do
|
53
|
-
subject.trim_start( Date.today.jd ).attributes[:trim_start].should eq Date.today.jd
|
54
|
-
end
|
55
|
-
|
56
|
-
end
|
57
|
-
|
58
|
-
end
|
59
|
-
|
60
|
-
describe "#trim_end" do
|
61
|
-
|
62
|
-
subject{ dataset.data }
|
63
|
-
|
64
|
-
context "given invalid date" do
|
65
|
-
|
66
|
-
it "rejects string" do
|
67
|
-
subject.trim_end('invalid').attributes[:trim_end].should be_nil
|
68
|
-
end
|
69
|
-
|
70
|
-
end
|
71
|
-
context "given valid date" do
|
72
|
-
|
73
|
-
it "accepts string" do
|
74
|
-
subject.trim_end('1980-01-01').attributes[:trim_end].should eq Date.parse('1980-01-01').jd
|
75
|
-
end
|
76
|
-
|
77
|
-
it "accepts julian date" do
|
78
|
-
subject.trim_end( Date.today.jd ).attributes[:trim_end].should eq Date.today.jd
|
79
|
-
end
|
80
|
-
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
end
|
@@ -1,44 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require 'spec_helper'
|
3
|
-
|
4
|
-
describe Quandl::Cassandra::Dataset do
|
5
|
-
|
6
|
-
let(:dataset){ create(:dataset) }
|
7
|
-
subject { dataset }
|
8
|
-
|
9
|
-
it "should have columns" do
|
10
|
-
subject.columns.count.should eq 4
|
11
|
-
end
|
12
|
-
|
13
|
-
describe "#data" do
|
14
|
-
|
15
|
-
subject{ dataset.data }
|
16
|
-
|
17
|
-
its(:class){ should eq Quandl::Cassandra::Data::ScopeScope }
|
18
|
-
|
19
|
-
its(:to_date){ should be_present }
|
20
|
-
its(:to_h){ should be_present }
|
21
|
-
|
22
|
-
it "should have data" do
|
23
|
-
subject.count.should eq 60
|
24
|
-
end
|
25
|
-
|
26
|
-
it "should collapse to daily" do
|
27
|
-
subject.collapse(:daily).count.should eq 60
|
28
|
-
end
|
29
|
-
|
30
|
-
it "should collapse to weekly" do
|
31
|
-
subject.collapse(:weekly).count.should be < 12
|
32
|
-
end
|
33
|
-
|
34
|
-
it "should collapse to monthly" do
|
35
|
-
subject.collapse(:monthly).count.should be < 4
|
36
|
-
end
|
37
|
-
|
38
|
-
it "should collapse to annual" do
|
39
|
-
subject.collapse(:annual).count.should eq 1
|
40
|
-
end
|
41
|
-
|
42
|
-
end
|
43
|
-
|
44
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require 'spec_helper'
|
3
|
-
|
4
|
-
describe Quandl::Cassandra::Dataset do
|
5
|
-
|
6
|
-
context "column" do
|
7
|
-
|
8
|
-
subject { create(:dataset) }
|
9
|
-
|
10
|
-
it "should return first column" do
|
11
|
-
subject.data.column(1)[0][1].should eq subject.data.scoped.to_table[0][1]
|
12
|
-
end
|
13
|
-
|
14
|
-
it "should return second column" do
|
15
|
-
subject.data.column(2)[0][1].should eq subject.data.scoped.to_table[0][2]
|
16
|
-
end
|
17
|
-
|
18
|
-
it "should return third column" do
|
19
|
-
subject.data.column(2)[0][1].should eq subject.data.scoped.to_table[0][2]
|
20
|
-
end
|
21
|
-
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require 'spec_helper'
|
3
|
-
|
4
|
-
describe Quandl::Cassandra::Dataset do
|
5
|
-
|
6
|
-
let(:data){ Quandl::Fabricate::Data.rand( rows: 4, columns: 4, nils: false ) }
|
7
|
-
let(:dataset){ create( :dataset, data: data ) }
|
8
|
-
|
9
|
-
subject{ dataset }
|
10
|
-
|
11
|
-
its(:dataset_attribute){ should be_a DatasetAttribute }
|
12
|
-
its(:trim_start){ should eq Date.jd(Dataset.find(dataset.id).data[-1][0]) }
|
13
|
-
its(:trim_end){ should eq Date.jd(Dataset.find(dataset.id).data[0][0]) }
|
14
|
-
its(:updated_at){ should_not be_nil }
|
15
|
-
|
16
|
-
context "after save" do
|
17
|
-
before(:each){
|
18
|
-
@previously_updated_at = subject.updated_at
|
19
|
-
subject.save
|
20
|
-
}
|
21
|
-
its(:updated_at){ should_not eq @previously_updated_at }
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
@@ -1,26 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require 'spec_helper'
|
3
|
-
|
4
|
-
describe Quandl::Cassandra::Dataset do
|
5
|
-
|
6
|
-
let(:data){ Quandl::Fabricate::Data.rand( rows: 216, columns: 2, frequency: :weekly, nils: false ).to_csv }
|
7
|
-
let(:dataset){ create( :dataset, data: data ) }
|
8
|
-
subject { dataset }
|
9
|
-
|
10
|
-
it "should be monthly" do
|
11
|
-
subject.frequency.should eq 'weekly'
|
12
|
-
end
|
13
|
-
|
14
|
-
it "should pluck the second row" do
|
15
|
-
subject.data.scoped.collapse(:monthly).row(1).to_a.should eq [subject.data.scoped.collapse(:monthly)[1]]
|
16
|
-
end
|
17
|
-
|
18
|
-
it "should collapse and pluck the second year" do
|
19
|
-
subject.data.scoped.collapse(:annual).row(2).to_a.should eq [subject.data.scoped.collapse(:annual)[2]]
|
20
|
-
end
|
21
|
-
|
22
|
-
it "should collapse and pluck the second year with a transformation" do
|
23
|
-
subject.data.scoped.collapse(:annual).transform(:rdiff).row(2).to_a.should eq [subject.data.scoped.collapse(:annual).transform(:rdiff)[2]]
|
24
|
-
end
|
25
|
-
|
26
|
-
end
|
@@ -1,16 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require 'spec_helper'
|
3
|
-
|
4
|
-
describe Dataset do
|
5
|
-
|
6
|
-
let(:data){ Quandl::Data.new([ [1002, 10], [1001, 20], [1000, 30] ]) }
|
7
|
-
let(:dataset){ build(:dataset) }
|
8
|
-
|
9
|
-
it "should cumul data" do
|
10
|
-
dataset.data = data
|
11
|
-
dataset.save!
|
12
|
-
cumul_data = Dataset.find(dataset.id).data.transform(:cumul).to_table.data_array
|
13
|
-
cumul_data.should eq [ [1002, 60.0], [1001, 50.0], [1000, 30.0] ]
|
14
|
-
end
|
15
|
-
|
16
|
-
end
|