quandl_cassandra 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/quandl/cassandra/base/attributes.rb +4 -4
- data/lib/quandl/cassandra/base/persistence.rb +24 -1
- data/lib/quandl/cassandra/base/sanitization.rb +4 -1
- data/lib/quandl/cassandra/base/schema.rb +8 -0
- data/lib/quandl/cassandra/base/scoping.rb +47 -3
- data/lib/quandl/cassandra/types/boolean_type.rb +1 -1
- data/lib/quandl/cassandra/types/decimal_type.rb +1 -1
- data/lib/quandl/cassandra/types/double_type.rb +1 -1
- data/lib/quandl/cassandra/types/float_type.rb +1 -1
- data/lib/quandl/cassandra/types/integer_type.rb +1 -1
- data/lib/quandl/cassandra/types/long_type.rb +1 -1
- data/lib/quandl/cassandra/types/timestamp_type.rb +1 -1
- data/lib/quandl/cassandra/types/utf8_type.rb +1 -1
- data/lib/quandl/cassandra/types/uuid_type.rb +1 -1
- data/lib/quandl/cassandra/version.rb +1 -1
- data/lib/quandl/cassandra_models/column/read/collapse.rb +2 -1
- data/lib/quandl/cassandra_models/column/read/column.rb +3 -2
- data/lib/quandl/cassandra_models/column/read/{data_table.rb → data.rb} +6 -4
- data/lib/quandl/cassandra_models/column/read/offset.rb +4 -0
- data/lib/quandl/cassandra_models/column/read/{query.rb → select_columns.rb} +21 -4
- data/lib/quandl/cassandra_models/column/read.rb +9 -5
- data/lib/quandl/cassandra_models/column/write/insert_column_attributes.rb +22 -0
- data/lib/quandl/cassandra_models/column/write/insert_columns.rb +29 -12
- data/lib/quandl/cassandra_models/column/write.rb +2 -2
- data/lib/quandl/cassandra_models/data/search.rb +104 -0
- data/lib/quandl/cassandra_models/data.rb +12 -46
- data/lib/quandl/cassandra_models/dataset.rb +17 -8
- data/lib/quandl/cassandra_models/multiset.rb +3 -3
- data/quandl_cassandra.gemspec +1 -1
- data/spec/lib/quandl/cassandra/base/sanitization_spec.rb +7 -0
- data/spec/lib/quandl/cassandra/base/scoping_spec.rb +11 -0
- data/spec/lib/quandl/cassandra_models/column_spec.rb +1 -1
- data/spec/lib/quandl/cassandra_models/data_spec.rb +51 -1
- data/spec/lib/quandl/cassandra_models/dataset/collapse_spec.rb +3 -0
- data/spec/lib/quandl/cassandra_models/dataset/column_spec.rb +3 -4
- data/spec/lib/quandl/cassandra_models/dataset/persistence_spec.rb +2 -2
- data/spec/lib/quandl/cassandra_models/dataset/row_spec.rb +8 -8
- data/spec/lib/quandl/cassandra_models/dataset/trim_spec.rb +6 -6
- data/spec/lib/quandl/cassandra_models/dataset/update_spec.rb +4 -4
- data/spec/lib/quandl/cassandra_models/dataset_spec.rb +42 -3
- data/spec/lib/quandl/cassandra_models/multiset/transform_spec.rb +2 -1
- metadata +10 -7
- data/lib/quandl/cassandra_models/column/write/insert_data.rb +0 -39
@@ -1,52 +1,18 @@
|
|
1
|
-
class Quandl::Cassandra::Data
|
1
|
+
class Quandl::Cassandra::Data < Quandl::Data
|
2
2
|
|
3
|
-
|
3
|
+
require_relative 'data/search'
|
4
4
|
|
5
|
-
|
5
|
+
include Quandl::Cassandra::Data::Search
|
6
6
|
|
7
|
-
|
7
|
+
attr_accessor :dataset_id, :column_ids, :column_frequencies
|
8
8
|
|
9
|
-
scope
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
scope :collapse, ->(v){ where( collapse: v.to_sym ) if Quandl::Operation::Collapse.valid_collapse?(v) }
|
17
|
-
scope :transform, ->(v){ where( transform: v.to_sym ) if Quandl::Operation::Transform.valid_transformation?(v) }
|
18
|
-
|
19
|
-
scope :order, ->(v){
|
20
|
-
order = ( v.to_sym == :asc ) ? :asc : :desc
|
21
|
-
where( order: order )
|
22
|
-
}
|
23
|
-
|
24
|
-
scope :trim_start, ->(date){ where( trim_start: parse_date(date).jd ) }
|
25
|
-
scope :trim_end, ->(date){ where( trim_end: parse_date(date).jd ) }
|
26
|
-
|
27
|
-
scope_helper :find, ->(id){ id(id).to_table }
|
28
|
-
scope_helper :to_table, ->{ all }
|
29
|
-
|
30
|
-
scope_helper :parse_date, ->( date ){
|
31
|
-
begin
|
32
|
-
date = Date.jd(date.to_i) if date.kind_of?(String) && date.numeric?
|
33
|
-
date = Date.jd(date) if date.is_a?(Integer)
|
34
|
-
date = Date.parse(date) if date.is_a?(String) && date =~ /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/
|
35
|
-
date
|
36
|
-
rescue
|
37
|
-
nil
|
38
|
-
end
|
39
|
-
}
|
40
|
-
|
41
|
-
scope.class_eval do
|
42
|
-
|
43
|
-
delegate :inspect, :==, to: :all, allow_nil: true
|
44
|
-
Array.forwardable_methods.each{|mname| delegate(mname, to: :all, allow_nil: true ) unless self.respond_to?(mname) }
|
45
|
-
|
46
|
-
def all
|
47
|
-
@all ||= Quandl::Cassandra::Column.read( attributes.merge(scope_attributes) )
|
48
|
-
end
|
49
|
-
|
9
|
+
# start a new scope from this data
|
10
|
+
def scoped
|
11
|
+
s = self.class.scope.new
|
12
|
+
s.id(dataset_id)
|
13
|
+
s.column_ids(column_ids) if column_ids
|
14
|
+
s.column_frequencies(column_frequencies) if column_frequencies
|
15
|
+
s
|
50
16
|
end
|
51
|
-
|
17
|
+
|
52
18
|
end
|
@@ -6,12 +6,12 @@ class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
|
|
6
6
|
define_attributes :id, :data, :column_ids
|
7
7
|
|
8
8
|
before_save :save_columns, :save_data, :save_dataset_attribute
|
9
|
-
after_save :clear_attributes
|
9
|
+
after_save :clear_attributes!
|
10
10
|
|
11
11
|
delegate :type, :updated_at, :created_at, :frequency, to: :dataset_attribute, allow_nil: true
|
12
12
|
|
13
13
|
def self.find_column_ids_by_id(id)
|
14
|
-
Dataset.where( id: id ).pluck(:column_id, :position).sort_by{|r| r[1] }.collect{|r| r[0] }
|
14
|
+
Quandl::Cassandra::Dataset.where( id: id ).pluck(:column_id, :position).sort_by{|r| r[1] }.collect{|r| r[0] }
|
15
15
|
end
|
16
16
|
|
17
17
|
def column_attributes=(column_attrs)
|
@@ -26,15 +26,19 @@ class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
|
|
26
26
|
end
|
27
27
|
|
28
28
|
def column_ids
|
29
|
-
@column_ids ||= self.class.find_column_ids_by_id(id)
|
29
|
+
@column_ids ||= id.blank? ? [] : self.class.find_column_ids_by_id(id)
|
30
30
|
end
|
31
31
|
|
32
32
|
def trim_start
|
33
|
-
@trim_start ||=
|
33
|
+
@trim_start ||= Date.jd( data.scoped.limit(1).order(:asc)[0][0] )
|
34
|
+
rescue
|
35
|
+
nil
|
34
36
|
end
|
35
37
|
|
36
38
|
def trim_end
|
37
|
-
@trim_end ||=
|
39
|
+
@trim_end ||= Date.jd( data.scoped.limit(1).order(:desc)[0][0] )
|
40
|
+
rescue
|
41
|
+
nil
|
38
42
|
end
|
39
43
|
|
40
44
|
def data
|
@@ -46,7 +50,8 @@ class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
|
|
46
50
|
|
47
51
|
def data=(rows)
|
48
52
|
rows = Quandl::Data.new(rows) unless rows.is_a?(Quandl::Data)
|
49
|
-
|
53
|
+
data_will_change!
|
54
|
+
@attributes[:data] = rows
|
50
55
|
end
|
51
56
|
|
52
57
|
def data_scope
|
@@ -57,6 +62,10 @@ class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
|
|
57
62
|
@dataset_attribute ||= Quandl::Cassandra::DatasetAttribute.find_or_build(id)
|
58
63
|
end
|
59
64
|
|
65
|
+
def reload
|
66
|
+
clear_attributes!
|
67
|
+
end
|
68
|
+
|
60
69
|
protected
|
61
70
|
|
62
71
|
def save_dataset_attribute
|
@@ -72,12 +81,12 @@ class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
|
|
72
81
|
Quandl::Cassandra::Column.write( id: id, data: data ) if data_changed?
|
73
82
|
end
|
74
83
|
|
75
|
-
def clear_attributes
|
84
|
+
def clear_attributes!
|
76
85
|
@trim_start = nil
|
77
86
|
@trim_end = nil
|
78
87
|
@columns = nil
|
79
88
|
@column_ids = nil
|
80
|
-
@attributes
|
89
|
+
@attributes = { id: id }
|
81
90
|
end
|
82
91
|
|
83
92
|
end
|
@@ -26,10 +26,10 @@ class Quandl::Cassandra::Multiset < Quandl::Cassandra::Dataset
|
|
26
26
|
|
27
27
|
def column_ids_from_datasets_columns
|
28
28
|
ids = []
|
29
|
-
datasets_columns.split(',').each do |dataset_column|
|
29
|
+
datasets_columns.to_s.split(',').each do |dataset_column|
|
30
30
|
dataset_id, column = dataset_column.split('.')
|
31
|
-
datasets[dataset_id] ||= Quandl::Cassandra::Dataset.find(dataset_id).column_ids
|
32
|
-
ids << datasets[dataset_id][ column.to_i - 1 ]
|
31
|
+
datasets[dataset_id] ||= Quandl::Cassandra::Dataset.find(dataset_id).try(:column_ids)
|
32
|
+
ids << datasets[dataset_id][ column.to_i - 1 ] if datasets[dataset_id].is_a?(Array)
|
33
33
|
end
|
34
34
|
ids
|
35
35
|
end
|
data/quandl_cassandra.gemspec
CHANGED
@@ -26,7 +26,7 @@ Gem::Specification.new do |s|
|
|
26
26
|
s.add_runtime_dependency "activesupport", ">= 3.0.0"
|
27
27
|
s.add_runtime_dependency "activemodel", ">= 3.0.0"
|
28
28
|
|
29
|
-
s.add_runtime_dependency "scope_composer", "~> 0.
|
29
|
+
s.add_runtime_dependency "scope_composer", "~> 0.4"
|
30
30
|
s.add_runtime_dependency "quandl_data", "~> 1.0"
|
31
31
|
s.add_runtime_dependency "quandl_logger", "~> 0.1"
|
32
32
|
s.add_runtime_dependency "quandl_operation", "~> 0.1"
|
@@ -10,6 +10,17 @@ describe Quandl::Cassandra::Base::Scoping do
|
|
10
10
|
it{ should respond_to name }
|
11
11
|
end
|
12
12
|
|
13
|
+
its(:scope_names){ should eq TestModel.scope.scope_names }
|
14
|
+
|
15
|
+
describe ".find_or_build" do
|
16
|
+
it "given nil should be a new record" do
|
17
|
+
Dataset.find_or_build(nil).new_record?.should be_true
|
18
|
+
end
|
19
|
+
it "given empty sting should be a new record" do
|
20
|
+
Dataset.find_or_build('').new_record?.should be_true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
13
24
|
describe ".to_cql" do
|
14
25
|
|
15
26
|
let(:scope) { TestModel.scope.new }
|
@@ -3,7 +3,7 @@ require 'spec_helper'
|
|
3
3
|
describe Quandl::Cassandra::Column do
|
4
4
|
|
5
5
|
let(:id){ rand(10000*10000) + 10000*10000 }
|
6
|
-
let(:data){ Quandl::Fabricate::Data.rand( rows: 10, columns: 2 ) }
|
6
|
+
let(:data){ Quandl::Fabricate::Data.rand( rows: 10, columns: 2, nils: false ) }
|
7
7
|
let(:dataset){ Quandl::Cassandra::Dataset.create( id: id, data: data ) }
|
8
8
|
|
9
9
|
describe ".read" do
|
@@ -30,5 +30,55 @@ describe Quandl::Cassandra::Data do
|
|
30
30
|
scope.attributes[:collapse].should eq :weekly
|
31
31
|
end
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
|
+
describe "#trim_start" do
|
35
|
+
|
36
|
+
subject{ dataset.data }
|
37
|
+
|
38
|
+
context "given invalid date" do
|
39
|
+
|
40
|
+
it "rejects string" do
|
41
|
+
subject.trim_start('invalid').attributes[:trim_start].should be_nil
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
|
46
|
+
context "given valid date" do
|
47
|
+
|
48
|
+
it "accepts string" do
|
49
|
+
subject.trim_start('1980-01-01').attributes[:trim_start].should eq Date.parse('1980-01-01').jd
|
50
|
+
end
|
51
|
+
|
52
|
+
it "accepts julian date" do
|
53
|
+
subject.trim_start( Date.today.jd ).attributes[:trim_start].should eq Date.today.jd
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
describe "#trim_end" do
|
61
|
+
|
62
|
+
subject{ dataset.data }
|
63
|
+
|
64
|
+
context "given invalid date" do
|
65
|
+
|
66
|
+
it "rejects string" do
|
67
|
+
subject.trim_end('invalid').attributes[:trim_end].should be_nil
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
context "given valid date" do
|
72
|
+
|
73
|
+
it "accepts string" do
|
74
|
+
subject.trim_end('1980-01-01').attributes[:trim_end].should eq Date.parse('1980-01-01').jd
|
75
|
+
end
|
76
|
+
|
77
|
+
it "accepts julian date" do
|
78
|
+
subject.trim_end( Date.today.jd ).attributes[:trim_end].should eq Date.today.jd
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
34
84
|
end
|
@@ -8,16 +8,15 @@ describe Quandl::Cassandra::Dataset do
|
|
8
8
|
subject { create(:dataset) }
|
9
9
|
|
10
10
|
it "should return first column" do
|
11
|
-
subject.data.column(1)[0][1].should eq subject.
|
11
|
+
subject.data.column(1)[0][1].should eq subject.data.scoped.to_table[0][1]
|
12
12
|
end
|
13
13
|
|
14
14
|
it "should return second column" do
|
15
|
-
|
16
|
-
subject.data.column(2)[0][1].should eq subject.data_scope.to_table[0][2]
|
15
|
+
subject.data.column(2)[0][1].should eq subject.data.scoped.to_table[0][2]
|
17
16
|
end
|
18
17
|
|
19
18
|
it "should return third column" do
|
20
|
-
subject.data.column(2)[0][1].should eq subject.
|
19
|
+
subject.data.column(2)[0][1].should eq subject.data.scoped.to_table[0][2]
|
21
20
|
end
|
22
21
|
|
23
22
|
end
|
@@ -9,8 +9,8 @@ describe Quandl::Cassandra::Dataset do
|
|
9
9
|
subject{ dataset }
|
10
10
|
|
11
11
|
its(:dataset_attribute){ should be_a DatasetAttribute }
|
12
|
-
its(:trim_start){ should eq Dataset.find(dataset.id).data[-1][0] }
|
13
|
-
its(:trim_end){ should eq Dataset.find(dataset.id).data[0][0] }
|
12
|
+
its(:trim_start){ should eq Date.jd(Dataset.find(dataset.id).data[-1][0]) }
|
13
|
+
its(:trim_end){ should eq Date.jd(Dataset.find(dataset.id).data[0][0]) }
|
14
14
|
its(:updated_at){ should_not be_nil }
|
15
15
|
|
16
16
|
context "after save" do
|
@@ -3,24 +3,24 @@ require 'spec_helper'
|
|
3
3
|
|
4
4
|
describe Quandl::Cassandra::Dataset do
|
5
5
|
|
6
|
-
let(:data){ Quandl::Fabricate::Data.rand( rows:
|
6
|
+
let(:data){ Quandl::Fabricate::Data.rand( rows: 216, columns: 2, frequency: :weekly, nils: false ).to_csv }
|
7
7
|
let(:dataset){ create( :dataset, data: data ) }
|
8
8
|
subject { dataset }
|
9
9
|
|
10
10
|
it "should be monthly" do
|
11
|
-
subject.frequency.should eq '
|
11
|
+
subject.frequency.should eq 'weekly'
|
12
12
|
end
|
13
|
-
|
13
|
+
|
14
14
|
it "should pluck the second row" do
|
15
|
-
subject.
|
15
|
+
subject.data.scoped.collapse(:monthly).row(1).to_a.should eq [subject.data.scoped.collapse(:monthly)[1]]
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
it "should collapse and pluck the second year" do
|
19
|
-
subject.
|
19
|
+
subject.data.scoped.collapse(:annual).row(2).to_a.should eq [subject.data.scoped.collapse(:annual)[2]]
|
20
20
|
end
|
21
|
-
|
21
|
+
|
22
22
|
it "should collapse and pluck the second year with a transformation" do
|
23
|
-
subject.
|
23
|
+
subject.data.scoped.collapse(:annual).transform(:rdiff).row(2).to_a.should eq [subject.data.scoped.collapse(:annual).transform(:rdiff)[2]]
|
24
24
|
end
|
25
25
|
|
26
26
|
end
|
@@ -11,7 +11,7 @@ describe Quandl::Cassandra::Dataset do
|
|
11
11
|
|
12
12
|
it "should include trim_start" do
|
13
13
|
# raw data
|
14
|
-
source_data = subject.
|
14
|
+
source_data = subject.data.scoped.collapse(collapse_to).to_table
|
15
15
|
trim_start = source_data[-1][0]
|
16
16
|
# trim and check
|
17
17
|
trim_data = subject.data.trim_start( trim_start ).collapse(collapse_to).to_table
|
@@ -19,7 +19,7 @@ describe Quandl::Cassandra::Dataset do
|
|
19
19
|
end
|
20
20
|
it "should include trim_end" do
|
21
21
|
# raw data
|
22
|
-
source_data = subject.
|
22
|
+
source_data = subject.data.scoped.collapse(collapse_to).to_table
|
23
23
|
trim_end = source_data[1][0]
|
24
24
|
# trim and check
|
25
25
|
trim_data = subject.data.trim_end( trim_end ).collapse(collapse_to).to_table
|
@@ -27,7 +27,7 @@ describe Quandl::Cassandra::Dataset do
|
|
27
27
|
end
|
28
28
|
it "should include trim_start and trim_end" do
|
29
29
|
# raw data
|
30
|
-
source_data = subject.
|
30
|
+
source_data = subject.data.scoped.collapse(collapse_to).to_table
|
31
31
|
trim_end = source_data[1][0]
|
32
32
|
trim_start = source_data[-1][0]
|
33
33
|
# trim and check
|
@@ -40,7 +40,7 @@ describe Quandl::Cassandra::Dataset do
|
|
40
40
|
context "when transformed to #{transformed_to}" do
|
41
41
|
it "should include trim_start" do
|
42
42
|
# raw data
|
43
|
-
source_data = subject.
|
43
|
+
source_data = subject.data.scoped.collapse(collapse_to).transform(transformed_to).to_table
|
44
44
|
trim_start = source_data[-1][0]
|
45
45
|
# trim and check
|
46
46
|
trim_data = subject.data.trim_start( trim_start ).transform(transformed_to).collapse(collapse_to).to_table
|
@@ -48,14 +48,14 @@ describe Quandl::Cassandra::Dataset do
|
|
48
48
|
end
|
49
49
|
it "should include trim_end" do
|
50
50
|
# raw data
|
51
|
-
source_data = subject.
|
51
|
+
source_data = subject.data.scoped.collapse(collapse_to).transform(transformed_to).to_table
|
52
52
|
trim_end = source_data[-1][0]
|
53
53
|
# trim and check
|
54
54
|
trim_data = subject.data.trim_end( trim_end ).transform(transformed_to).collapse(collapse_to).to_table
|
55
55
|
trim_data.first[0].should eq trim_end
|
56
56
|
end
|
57
57
|
it "should include trim_start and trim_end" do
|
58
|
-
source_data = subject.
|
58
|
+
source_data = subject.data.scoped.collapse(collapse_to).transform(transformed_to).to_table
|
59
59
|
trim_end = source_data[1][0]
|
60
60
|
trim_start = source_data[-1][0]
|
61
61
|
# trim and check
|
@@ -8,8 +8,8 @@ describe Dataset do
|
|
8
8
|
subject{ create(:dataset, data: Quandl::Fabricate::Data.rand( rows: 730, columns: 2, nils: false ).to_csv ) }
|
9
9
|
|
10
10
|
it "should update the collapse data" do
|
11
|
-
old_row = subject.
|
12
|
-
old_row_month = subject.
|
11
|
+
old_row = subject.data.scoped[0]
|
12
|
+
old_row_month = subject.data.scoped.collapse(:monthly)[0]
|
13
13
|
# update
|
14
14
|
dataset = Dataset.find(subject.id)
|
15
15
|
# advance data dates by 60 days
|
@@ -22,8 +22,8 @@ describe Dataset do
|
|
22
22
|
dataset.data = new_data.to_a.collect{|r| r.to_csv }.join
|
23
23
|
dataset.save!
|
24
24
|
|
25
|
-
new_row = dataset.
|
26
|
-
new_row_month = dataset.
|
25
|
+
new_row = dataset.data.scoped[0]
|
26
|
+
new_row_month = dataset.data.scoped.collapse(:monthly)[0]
|
27
27
|
|
28
28
|
new_row[0].should_not eq new_row_month[0]
|
29
29
|
new_row[1].should eq new_row_month[1]
|
@@ -13,12 +13,35 @@ describe Quandl::Cassandra::Dataset do
|
|
13
13
|
its(:id){ should eq id }
|
14
14
|
its(:changes){ should eq( { "id" => [nil, id] }) }
|
15
15
|
|
16
|
+
describe ".new" do
|
17
|
+
let(:dataset){ Quandl::Cassandra::Dataset.new( id: 0 ) }
|
18
|
+
subject{ dataset }
|
19
|
+
its(:data){ should eq [] }
|
20
|
+
its(:column_ids){ should eq [] }
|
21
|
+
its(:columns){ should eq [] }
|
22
|
+
describe "#data" do
|
23
|
+
subject{ dataset.data }
|
24
|
+
its(:count){ should eq 0 }
|
25
|
+
its(:to_a){ should eq Quandl::Cassandra::Data.new }
|
26
|
+
context "given filters" do
|
27
|
+
subject{ dataset.data.row(0).column(1) }
|
28
|
+
its(:to_a){ should eq Quandl::Cassandra::Data.new }
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe "#save" do
|
34
|
+
subject{ Quandl::Cassandra::Dataset.new }
|
35
|
+
before(:each){ subject.save }
|
36
|
+
its(:new_record?){ should be_true }
|
37
|
+
end
|
38
|
+
|
16
39
|
describe "#class" do
|
17
40
|
subject{ dataset.class }
|
18
41
|
its(:table_name){ should eq 'datasets' }
|
19
42
|
end
|
20
43
|
|
21
|
-
|
44
|
+
describe "#save" do
|
22
45
|
|
23
46
|
before(:each){
|
24
47
|
dataset.data = Quandl::Fabricate::Data.rand(rows: 10, columns: 2, nils: false)
|
@@ -31,7 +54,11 @@ describe Quandl::Cassandra::Dataset do
|
|
31
54
|
|
32
55
|
describe ".find" do
|
33
56
|
subject{ Quandl::Cassandra::Dataset.find(id) }
|
34
|
-
its(:data){ should eq dataset.data }
|
57
|
+
its(:data){ should eq dataset.data.to_table }
|
58
|
+
it "data should count and return data" do
|
59
|
+
subject.data.count.should eq 10
|
60
|
+
subject.data.to_table.should be_a Quandl::Cassandra::Data
|
61
|
+
end
|
35
62
|
it "columns should eq dataset.columns" do
|
36
63
|
subject.columns.collect{|c| c.id.to_s }.should eq dataset.columns.collect{|c| c.id.to_s }
|
37
64
|
end
|
@@ -40,6 +67,19 @@ describe Quandl::Cassandra::Dataset do
|
|
40
67
|
end
|
41
68
|
end
|
42
69
|
|
70
|
+
describe "#reload" do
|
71
|
+
before(:each){
|
72
|
+
dataset.data.limit(5).to_a
|
73
|
+
dataset.reload
|
74
|
+
}
|
75
|
+
|
76
|
+
describe "#attributes" do
|
77
|
+
subject{ dataset.attributes }
|
78
|
+
its([:data]){ should eq nil }
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
|
43
83
|
describe "#data" do
|
44
84
|
subject{ dataset.data }
|
45
85
|
its(:count){ should eq 10 }
|
@@ -57,7 +97,6 @@ describe Quandl::Cassandra::Dataset do
|
|
57
97
|
its(:first){ should be_a Quandl::Cassandra::ColumnAttribute }
|
58
98
|
end
|
59
99
|
|
60
|
-
|
61
100
|
end
|
62
101
|
|
63
102
|
end
|
@@ -8,7 +8,8 @@ describe Multiset do
|
|
8
8
|
let(:d1){ create(:dataset) }
|
9
9
|
let(:d2){ create(:dataset) }
|
10
10
|
subject { Multiset.with_columns("#{d2.id.to_i}.1,#{d1.id.to_i}.1,#{d2.id.to_i}.3,#{d1.id.to_i}.2") }
|
11
|
-
|
11
|
+
|
12
|
+
let(:data_table){ subject.data.scoped.to_table }
|
12
13
|
|
13
14
|
it "should change the scope" do
|
14
15
|
scope = subject.data.transform('rdiff')
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: quandl_cassandra
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-10-
|
12
|
+
date: 2013-10-31 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -130,7 +130,7 @@ dependencies:
|
|
130
130
|
requirements:
|
131
131
|
- - ~>
|
132
132
|
- !ruby/object:Gem::Version
|
133
|
-
version: '0.
|
133
|
+
version: '0.4'
|
134
134
|
type: :runtime
|
135
135
|
prerelease: false
|
136
136
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -138,7 +138,7 @@ dependencies:
|
|
138
138
|
requirements:
|
139
139
|
- - ~>
|
140
140
|
- !ruby/object:Gem::Version
|
141
|
-
version: '0.
|
141
|
+
version: '0.4'
|
142
142
|
- !ruby/object:Gem::Dependency
|
143
143
|
name: quandl_data
|
144
144
|
requirement: !ruby/object:Gem::Requirement
|
@@ -245,19 +245,20 @@ files:
|
|
245
245
|
- lib/quandl/cassandra_models/column/read.rb
|
246
246
|
- lib/quandl/cassandra_models/column/read/collapse.rb
|
247
247
|
- lib/quandl/cassandra_models/column/read/column.rb
|
248
|
-
- lib/quandl/cassandra_models/column/read/
|
248
|
+
- lib/quandl/cassandra_models/column/read/data.rb
|
249
249
|
- lib/quandl/cassandra_models/column/read/offset.rb
|
250
|
-
- lib/quandl/cassandra_models/column/read/query.rb
|
251
250
|
- lib/quandl/cassandra_models/column/read/row.rb
|
251
|
+
- lib/quandl/cassandra_models/column/read/select_columns.rb
|
252
252
|
- lib/quandl/cassandra_models/column/read/transform.rb
|
253
253
|
- lib/quandl/cassandra_models/column/read/type.rb
|
254
254
|
- lib/quandl/cassandra_models/column/write.rb
|
255
255
|
- lib/quandl/cassandra_models/column/write/group_data_by_column.rb
|
256
256
|
- lib/quandl/cassandra_models/column/write/group_data_by_frequency.rb
|
257
|
+
- lib/quandl/cassandra_models/column/write/insert_column_attributes.rb
|
257
258
|
- lib/quandl/cassandra_models/column/write/insert_columns.rb
|
258
|
-
- lib/quandl/cassandra_models/column/write/insert_data.rb
|
259
259
|
- lib/quandl/cassandra_models/column_attribute.rb
|
260
260
|
- lib/quandl/cassandra_models/data.rb
|
261
|
+
- lib/quandl/cassandra_models/data/search.rb
|
261
262
|
- lib/quandl/cassandra_models/dataset.rb
|
262
263
|
- lib/quandl/cassandra_models/dataset_attribute.rb
|
263
264
|
- lib/quandl/cassandra_models/multiset.rb
|
@@ -266,6 +267,7 @@ files:
|
|
266
267
|
- spec/expectations/string.rb
|
267
268
|
- spec/expectations/time.rb
|
268
269
|
- spec/factories/dataset.rb
|
270
|
+
- spec/lib/quandl/cassandra/base/sanitization_spec.rb
|
269
271
|
- spec/lib/quandl/cassandra/base/scoping_spec.rb
|
270
272
|
- spec/lib/quandl/cassandra_models/column/write/group_data_by_frequency_spec.rb
|
271
273
|
- spec/lib/quandl/cassandra_models/column/write_spec.rb
|
@@ -316,6 +318,7 @@ test_files:
|
|
316
318
|
- spec/expectations/string.rb
|
317
319
|
- spec/expectations/time.rb
|
318
320
|
- spec/factories/dataset.rb
|
321
|
+
- spec/lib/quandl/cassandra/base/sanitization_spec.rb
|
319
322
|
- spec/lib/quandl/cassandra/base/scoping_spec.rb
|
320
323
|
- spec/lib/quandl/cassandra_models/column/write/group_data_by_frequency_spec.rb
|
321
324
|
- spec/lib/quandl/cassandra_models/column/write_spec.rb
|
@@ -1,39 +0,0 @@
|
|
1
|
-
class Quandl::Cassandra::Column::Write::InsertData < Quandl::Cassandra::Column::Write
|
2
|
-
|
3
|
-
def perform
|
4
|
-
insert_data_in_batches.collect(&:value)
|
5
|
-
end
|
6
|
-
|
7
|
-
def insert_data_in_batches
|
8
|
-
futures = []
|
9
|
-
statements = []
|
10
|
-
frequency_column_data.each do |frequency, column_data|
|
11
|
-
column_data.each do |column_id, rows|
|
12
|
-
rows.each do |time_value|
|
13
|
-
# collect statements
|
14
|
-
statements << statement( column_id, frequency, time_value[0], time_value[1] )
|
15
|
-
# after 30 statements are collected, execute a batch insert
|
16
|
-
if statements.count >= Quandl::Cassandra.configuration.batch_size
|
17
|
-
# collect the futures
|
18
|
-
futures << execute_async_batch(statements)
|
19
|
-
# clear statements
|
20
|
-
statements = []
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
# execute any remaining statements
|
26
|
-
futures << execute_async_batch(statements) if statements.count > 0
|
27
|
-
futures
|
28
|
-
end
|
29
|
-
|
30
|
-
def execute_async_batch(statements)
|
31
|
-
batch = %Q{BEGIN UNLOGGED BATCH\n#{statements.join("\n")}\nAPPLY BATCH;}
|
32
|
-
future = Quandl::Cassandra::Base.execute_async( batch )
|
33
|
-
end
|
34
|
-
|
35
|
-
def statement( id, type, time, value )
|
36
|
-
"INSERT INTO columns (id, type, time, value) VALUES (#{id}, '#{type}', #{time}, #{value})"
|
37
|
-
end
|
38
|
-
|
39
|
-
end
|