quandl_cassandra 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/quandl/cassandra/base/attributes.rb +4 -4
- data/lib/quandl/cassandra/base/persistence.rb +24 -1
- data/lib/quandl/cassandra/base/sanitization.rb +4 -1
- data/lib/quandl/cassandra/base/schema.rb +8 -0
- data/lib/quandl/cassandra/base/scoping.rb +47 -3
- data/lib/quandl/cassandra/types/boolean_type.rb +1 -1
- data/lib/quandl/cassandra/types/decimal_type.rb +1 -1
- data/lib/quandl/cassandra/types/double_type.rb +1 -1
- data/lib/quandl/cassandra/types/float_type.rb +1 -1
- data/lib/quandl/cassandra/types/integer_type.rb +1 -1
- data/lib/quandl/cassandra/types/long_type.rb +1 -1
- data/lib/quandl/cassandra/types/timestamp_type.rb +1 -1
- data/lib/quandl/cassandra/types/utf8_type.rb +1 -1
- data/lib/quandl/cassandra/types/uuid_type.rb +1 -1
- data/lib/quandl/cassandra/version.rb +1 -1
- data/lib/quandl/cassandra_models/column/read/collapse.rb +2 -1
- data/lib/quandl/cassandra_models/column/read/column.rb +3 -2
- data/lib/quandl/cassandra_models/column/read/{data_table.rb → data.rb} +6 -4
- data/lib/quandl/cassandra_models/column/read/offset.rb +4 -0
- data/lib/quandl/cassandra_models/column/read/{query.rb → select_columns.rb} +21 -4
- data/lib/quandl/cassandra_models/column/read.rb +9 -5
- data/lib/quandl/cassandra_models/column/write/insert_column_attributes.rb +22 -0
- data/lib/quandl/cassandra_models/column/write/insert_columns.rb +29 -12
- data/lib/quandl/cassandra_models/column/write.rb +2 -2
- data/lib/quandl/cassandra_models/data/search.rb +104 -0
- data/lib/quandl/cassandra_models/data.rb +12 -46
- data/lib/quandl/cassandra_models/dataset.rb +17 -8
- data/lib/quandl/cassandra_models/multiset.rb +3 -3
- data/quandl_cassandra.gemspec +1 -1
- data/spec/lib/quandl/cassandra/base/sanitization_spec.rb +7 -0
- data/spec/lib/quandl/cassandra/base/scoping_spec.rb +11 -0
- data/spec/lib/quandl/cassandra_models/column_spec.rb +1 -1
- data/spec/lib/quandl/cassandra_models/data_spec.rb +51 -1
- data/spec/lib/quandl/cassandra_models/dataset/collapse_spec.rb +3 -0
- data/spec/lib/quandl/cassandra_models/dataset/column_spec.rb +3 -4
- data/spec/lib/quandl/cassandra_models/dataset/persistence_spec.rb +2 -2
- data/spec/lib/quandl/cassandra_models/dataset/row_spec.rb +8 -8
- data/spec/lib/quandl/cassandra_models/dataset/trim_spec.rb +6 -6
- data/spec/lib/quandl/cassandra_models/dataset/update_spec.rb +4 -4
- data/spec/lib/quandl/cassandra_models/dataset_spec.rb +42 -3
- data/spec/lib/quandl/cassandra_models/multiset/transform_spec.rb +2 -1
- metadata +10 -7
- data/lib/quandl/cassandra_models/column/write/insert_data.rb +0 -39
@@ -1,52 +1,18 @@
|
|
1
|
-
class Quandl::Cassandra::Data
|
1
|
+
class Quandl::Cassandra::Data < Quandl::Data
|
2
2
|
|
3
|
-
|
3
|
+
require_relative 'data/search'
|
4
4
|
|
5
|
-
|
5
|
+
include Quandl::Cassandra::Data::Search
|
6
6
|
|
7
|
-
|
7
|
+
attr_accessor :dataset_id, :column_ids, :column_frequencies
|
8
8
|
|
9
|
-
scope
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
scope :collapse, ->(v){ where( collapse: v.to_sym ) if Quandl::Operation::Collapse.valid_collapse?(v) }
|
17
|
-
scope :transform, ->(v){ where( transform: v.to_sym ) if Quandl::Operation::Transform.valid_transformation?(v) }
|
18
|
-
|
19
|
-
scope :order, ->(v){
|
20
|
-
order = ( v.to_sym == :asc ) ? :asc : :desc
|
21
|
-
where( order: order )
|
22
|
-
}
|
23
|
-
|
24
|
-
scope :trim_start, ->(date){ where( trim_start: parse_date(date).jd ) }
|
25
|
-
scope :trim_end, ->(date){ where( trim_end: parse_date(date).jd ) }
|
26
|
-
|
27
|
-
scope_helper :find, ->(id){ id(id).to_table }
|
28
|
-
scope_helper :to_table, ->{ all }
|
29
|
-
|
30
|
-
scope_helper :parse_date, ->( date ){
|
31
|
-
begin
|
32
|
-
date = Date.jd(date.to_i) if date.kind_of?(String) && date.numeric?
|
33
|
-
date = Date.jd(date) if date.is_a?(Integer)
|
34
|
-
date = Date.parse(date) if date.is_a?(String) && date =~ /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/
|
35
|
-
date
|
36
|
-
rescue
|
37
|
-
nil
|
38
|
-
end
|
39
|
-
}
|
40
|
-
|
41
|
-
scope.class_eval do
|
42
|
-
|
43
|
-
delegate :inspect, :==, to: :all, allow_nil: true
|
44
|
-
Array.forwardable_methods.each{|mname| delegate(mname, to: :all, allow_nil: true ) unless self.respond_to?(mname) }
|
45
|
-
|
46
|
-
def all
|
47
|
-
@all ||= Quandl::Cassandra::Column.read( attributes.merge(scope_attributes) )
|
48
|
-
end
|
49
|
-
|
9
|
+
# start a new scope from this data
|
10
|
+
def scoped
|
11
|
+
s = self.class.scope.new
|
12
|
+
s.id(dataset_id)
|
13
|
+
s.column_ids(column_ids) if column_ids
|
14
|
+
s.column_frequencies(column_frequencies) if column_frequencies
|
15
|
+
s
|
50
16
|
end
|
51
|
-
|
17
|
+
|
52
18
|
end
|
@@ -6,12 +6,12 @@ class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
|
|
6
6
|
define_attributes :id, :data, :column_ids
|
7
7
|
|
8
8
|
before_save :save_columns, :save_data, :save_dataset_attribute
|
9
|
-
after_save :clear_attributes
|
9
|
+
after_save :clear_attributes!
|
10
10
|
|
11
11
|
delegate :type, :updated_at, :created_at, :frequency, to: :dataset_attribute, allow_nil: true
|
12
12
|
|
13
13
|
def self.find_column_ids_by_id(id)
|
14
|
-
Dataset.where( id: id ).pluck(:column_id, :position).sort_by{|r| r[1] }.collect{|r| r[0] }
|
14
|
+
Quandl::Cassandra::Dataset.where( id: id ).pluck(:column_id, :position).sort_by{|r| r[1] }.collect{|r| r[0] }
|
15
15
|
end
|
16
16
|
|
17
17
|
def column_attributes=(column_attrs)
|
@@ -26,15 +26,19 @@ class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
|
|
26
26
|
end
|
27
27
|
|
28
28
|
def column_ids
|
29
|
-
@column_ids ||= self.class.find_column_ids_by_id(id)
|
29
|
+
@column_ids ||= id.blank? ? [] : self.class.find_column_ids_by_id(id)
|
30
30
|
end
|
31
31
|
|
32
32
|
def trim_start
|
33
|
-
@trim_start ||=
|
33
|
+
@trim_start ||= Date.jd( data.scoped.limit(1).order(:asc)[0][0] )
|
34
|
+
rescue
|
35
|
+
nil
|
34
36
|
end
|
35
37
|
|
36
38
|
def trim_end
|
37
|
-
@trim_end ||=
|
39
|
+
@trim_end ||= Date.jd( data.scoped.limit(1).order(:desc)[0][0] )
|
40
|
+
rescue
|
41
|
+
nil
|
38
42
|
end
|
39
43
|
|
40
44
|
def data
|
@@ -46,7 +50,8 @@ class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
|
|
46
50
|
|
47
51
|
def data=(rows)
|
48
52
|
rows = Quandl::Data.new(rows) unless rows.is_a?(Quandl::Data)
|
49
|
-
|
53
|
+
data_will_change!
|
54
|
+
@attributes[:data] = rows
|
50
55
|
end
|
51
56
|
|
52
57
|
def data_scope
|
@@ -57,6 +62,10 @@ class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
|
|
57
62
|
@dataset_attribute ||= Quandl::Cassandra::DatasetAttribute.find_or_build(id)
|
58
63
|
end
|
59
64
|
|
65
|
+
def reload
|
66
|
+
clear_attributes!
|
67
|
+
end
|
68
|
+
|
60
69
|
protected
|
61
70
|
|
62
71
|
def save_dataset_attribute
|
@@ -72,12 +81,12 @@ class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
|
|
72
81
|
Quandl::Cassandra::Column.write( id: id, data: data ) if data_changed?
|
73
82
|
end
|
74
83
|
|
75
|
-
def clear_attributes
|
84
|
+
def clear_attributes!
|
76
85
|
@trim_start = nil
|
77
86
|
@trim_end = nil
|
78
87
|
@columns = nil
|
79
88
|
@column_ids = nil
|
80
|
-
@attributes
|
89
|
+
@attributes = { id: id }
|
81
90
|
end
|
82
91
|
|
83
92
|
end
|
@@ -26,10 +26,10 @@ class Quandl::Cassandra::Multiset < Quandl::Cassandra::Dataset
|
|
26
26
|
|
27
27
|
def column_ids_from_datasets_columns
|
28
28
|
ids = []
|
29
|
-
datasets_columns.split(',').each do |dataset_column|
|
29
|
+
datasets_columns.to_s.split(',').each do |dataset_column|
|
30
30
|
dataset_id, column = dataset_column.split('.')
|
31
|
-
datasets[dataset_id] ||= Quandl::Cassandra::Dataset.find(dataset_id).column_ids
|
32
|
-
ids << datasets[dataset_id][ column.to_i - 1 ]
|
31
|
+
datasets[dataset_id] ||= Quandl::Cassandra::Dataset.find(dataset_id).try(:column_ids)
|
32
|
+
ids << datasets[dataset_id][ column.to_i - 1 ] if datasets[dataset_id].is_a?(Array)
|
33
33
|
end
|
34
34
|
ids
|
35
35
|
end
|
data/quandl_cassandra.gemspec
CHANGED
@@ -26,7 +26,7 @@ Gem::Specification.new do |s|
|
|
26
26
|
s.add_runtime_dependency "activesupport", ">= 3.0.0"
|
27
27
|
s.add_runtime_dependency "activemodel", ">= 3.0.0"
|
28
28
|
|
29
|
-
s.add_runtime_dependency "scope_composer", "~> 0.
|
29
|
+
s.add_runtime_dependency "scope_composer", "~> 0.4"
|
30
30
|
s.add_runtime_dependency "quandl_data", "~> 1.0"
|
31
31
|
s.add_runtime_dependency "quandl_logger", "~> 0.1"
|
32
32
|
s.add_runtime_dependency "quandl_operation", "~> 0.1"
|
@@ -10,6 +10,17 @@ describe Quandl::Cassandra::Base::Scoping do
|
|
10
10
|
it{ should respond_to name }
|
11
11
|
end
|
12
12
|
|
13
|
+
its(:scope_names){ should eq TestModel.scope.scope_names }
|
14
|
+
|
15
|
+
describe ".find_or_build" do
|
16
|
+
it "given nil should be a new record" do
|
17
|
+
Dataset.find_or_build(nil).new_record?.should be_true
|
18
|
+
end
|
19
|
+
it "given empty sting should be a new record" do
|
20
|
+
Dataset.find_or_build('').new_record?.should be_true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
13
24
|
describe ".to_cql" do
|
14
25
|
|
15
26
|
let(:scope) { TestModel.scope.new }
|
@@ -3,7 +3,7 @@ require 'spec_helper'
|
|
3
3
|
describe Quandl::Cassandra::Column do
|
4
4
|
|
5
5
|
let(:id){ rand(10000*10000) + 10000*10000 }
|
6
|
-
let(:data){ Quandl::Fabricate::Data.rand( rows: 10, columns: 2 ) }
|
6
|
+
let(:data){ Quandl::Fabricate::Data.rand( rows: 10, columns: 2, nils: false ) }
|
7
7
|
let(:dataset){ Quandl::Cassandra::Dataset.create( id: id, data: data ) }
|
8
8
|
|
9
9
|
describe ".read" do
|
@@ -30,5 +30,55 @@ describe Quandl::Cassandra::Data do
|
|
30
30
|
scope.attributes[:collapse].should eq :weekly
|
31
31
|
end
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
|
+
describe "#trim_start" do
|
35
|
+
|
36
|
+
subject{ dataset.data }
|
37
|
+
|
38
|
+
context "given invalid date" do
|
39
|
+
|
40
|
+
it "rejects string" do
|
41
|
+
subject.trim_start('invalid').attributes[:trim_start].should be_nil
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
|
46
|
+
context "given valid date" do
|
47
|
+
|
48
|
+
it "accepts string" do
|
49
|
+
subject.trim_start('1980-01-01').attributes[:trim_start].should eq Date.parse('1980-01-01').jd
|
50
|
+
end
|
51
|
+
|
52
|
+
it "accepts julian date" do
|
53
|
+
subject.trim_start( Date.today.jd ).attributes[:trim_start].should eq Date.today.jd
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
describe "#trim_end" do
|
61
|
+
|
62
|
+
subject{ dataset.data }
|
63
|
+
|
64
|
+
context "given invalid date" do
|
65
|
+
|
66
|
+
it "rejects string" do
|
67
|
+
subject.trim_end('invalid').attributes[:trim_end].should be_nil
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
context "given valid date" do
|
72
|
+
|
73
|
+
it "accepts string" do
|
74
|
+
subject.trim_end('1980-01-01').attributes[:trim_end].should eq Date.parse('1980-01-01').jd
|
75
|
+
end
|
76
|
+
|
77
|
+
it "accepts julian date" do
|
78
|
+
subject.trim_end( Date.today.jd ).attributes[:trim_end].should eq Date.today.jd
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
34
84
|
end
|
@@ -8,16 +8,15 @@ describe Quandl::Cassandra::Dataset do
|
|
8
8
|
subject { create(:dataset) }
|
9
9
|
|
10
10
|
it "should return first column" do
|
11
|
-
subject.data.column(1)[0][1].should eq subject.
|
11
|
+
subject.data.column(1)[0][1].should eq subject.data.scoped.to_table[0][1]
|
12
12
|
end
|
13
13
|
|
14
14
|
it "should return second column" do
|
15
|
-
|
16
|
-
subject.data.column(2)[0][1].should eq subject.data_scope.to_table[0][2]
|
15
|
+
subject.data.column(2)[0][1].should eq subject.data.scoped.to_table[0][2]
|
17
16
|
end
|
18
17
|
|
19
18
|
it "should return third column" do
|
20
|
-
subject.data.column(2)[0][1].should eq subject.
|
19
|
+
subject.data.column(2)[0][1].should eq subject.data.scoped.to_table[0][2]
|
21
20
|
end
|
22
21
|
|
23
22
|
end
|
@@ -9,8 +9,8 @@ describe Quandl::Cassandra::Dataset do
|
|
9
9
|
subject{ dataset }
|
10
10
|
|
11
11
|
its(:dataset_attribute){ should be_a DatasetAttribute }
|
12
|
-
its(:trim_start){ should eq Dataset.find(dataset.id).data[-1][0] }
|
13
|
-
its(:trim_end){ should eq Dataset.find(dataset.id).data[0][0] }
|
12
|
+
its(:trim_start){ should eq Date.jd(Dataset.find(dataset.id).data[-1][0]) }
|
13
|
+
its(:trim_end){ should eq Date.jd(Dataset.find(dataset.id).data[0][0]) }
|
14
14
|
its(:updated_at){ should_not be_nil }
|
15
15
|
|
16
16
|
context "after save" do
|
@@ -3,24 +3,24 @@ require 'spec_helper'
|
|
3
3
|
|
4
4
|
describe Quandl::Cassandra::Dataset do
|
5
5
|
|
6
|
-
let(:data){ Quandl::Fabricate::Data.rand( rows:
|
6
|
+
let(:data){ Quandl::Fabricate::Data.rand( rows: 216, columns: 2, frequency: :weekly, nils: false ).to_csv }
|
7
7
|
let(:dataset){ create( :dataset, data: data ) }
|
8
8
|
subject { dataset }
|
9
9
|
|
10
10
|
it "should be monthly" do
|
11
|
-
subject.frequency.should eq '
|
11
|
+
subject.frequency.should eq 'weekly'
|
12
12
|
end
|
13
|
-
|
13
|
+
|
14
14
|
it "should pluck the second row" do
|
15
|
-
subject.
|
15
|
+
subject.data.scoped.collapse(:monthly).row(1).to_a.should eq [subject.data.scoped.collapse(:monthly)[1]]
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
it "should collapse and pluck the second year" do
|
19
|
-
subject.
|
19
|
+
subject.data.scoped.collapse(:annual).row(2).to_a.should eq [subject.data.scoped.collapse(:annual)[2]]
|
20
20
|
end
|
21
|
-
|
21
|
+
|
22
22
|
it "should collapse and pluck the second year with a transformation" do
|
23
|
-
subject.
|
23
|
+
subject.data.scoped.collapse(:annual).transform(:rdiff).row(2).to_a.should eq [subject.data.scoped.collapse(:annual).transform(:rdiff)[2]]
|
24
24
|
end
|
25
25
|
|
26
26
|
end
|
@@ -11,7 +11,7 @@ describe Quandl::Cassandra::Dataset do
|
|
11
11
|
|
12
12
|
it "should include trim_start" do
|
13
13
|
# raw data
|
14
|
-
source_data = subject.
|
14
|
+
source_data = subject.data.scoped.collapse(collapse_to).to_table
|
15
15
|
trim_start = source_data[-1][0]
|
16
16
|
# trim and check
|
17
17
|
trim_data = subject.data.trim_start( trim_start ).collapse(collapse_to).to_table
|
@@ -19,7 +19,7 @@ describe Quandl::Cassandra::Dataset do
|
|
19
19
|
end
|
20
20
|
it "should include trim_end" do
|
21
21
|
# raw data
|
22
|
-
source_data = subject.
|
22
|
+
source_data = subject.data.scoped.collapse(collapse_to).to_table
|
23
23
|
trim_end = source_data[1][0]
|
24
24
|
# trim and check
|
25
25
|
trim_data = subject.data.trim_end( trim_end ).collapse(collapse_to).to_table
|
@@ -27,7 +27,7 @@ describe Quandl::Cassandra::Dataset do
|
|
27
27
|
end
|
28
28
|
it "should include trim_start and trim_end" do
|
29
29
|
# raw data
|
30
|
-
source_data = subject.
|
30
|
+
source_data = subject.data.scoped.collapse(collapse_to).to_table
|
31
31
|
trim_end = source_data[1][0]
|
32
32
|
trim_start = source_data[-1][0]
|
33
33
|
# trim and check
|
@@ -40,7 +40,7 @@ describe Quandl::Cassandra::Dataset do
|
|
40
40
|
context "when transformed to #{transformed_to}" do
|
41
41
|
it "should include trim_start" do
|
42
42
|
# raw data
|
43
|
-
source_data = subject.
|
43
|
+
source_data = subject.data.scoped.collapse(collapse_to).transform(transformed_to).to_table
|
44
44
|
trim_start = source_data[-1][0]
|
45
45
|
# trim and check
|
46
46
|
trim_data = subject.data.trim_start( trim_start ).transform(transformed_to).collapse(collapse_to).to_table
|
@@ -48,14 +48,14 @@ describe Quandl::Cassandra::Dataset do
|
|
48
48
|
end
|
49
49
|
it "should include trim_end" do
|
50
50
|
# raw data
|
51
|
-
source_data = subject.
|
51
|
+
source_data = subject.data.scoped.collapse(collapse_to).transform(transformed_to).to_table
|
52
52
|
trim_end = source_data[-1][0]
|
53
53
|
# trim and check
|
54
54
|
trim_data = subject.data.trim_end( trim_end ).transform(transformed_to).collapse(collapse_to).to_table
|
55
55
|
trim_data.first[0].should eq trim_end
|
56
56
|
end
|
57
57
|
it "should include trim_start and trim_end" do
|
58
|
-
source_data = subject.
|
58
|
+
source_data = subject.data.scoped.collapse(collapse_to).transform(transformed_to).to_table
|
59
59
|
trim_end = source_data[1][0]
|
60
60
|
trim_start = source_data[-1][0]
|
61
61
|
# trim and check
|
@@ -8,8 +8,8 @@ describe Dataset do
|
|
8
8
|
subject{ create(:dataset, data: Quandl::Fabricate::Data.rand( rows: 730, columns: 2, nils: false ).to_csv ) }
|
9
9
|
|
10
10
|
it "should update the collapse data" do
|
11
|
-
old_row = subject.
|
12
|
-
old_row_month = subject.
|
11
|
+
old_row = subject.data.scoped[0]
|
12
|
+
old_row_month = subject.data.scoped.collapse(:monthly)[0]
|
13
13
|
# update
|
14
14
|
dataset = Dataset.find(subject.id)
|
15
15
|
# advance data dates by 60 days
|
@@ -22,8 +22,8 @@ describe Dataset do
|
|
22
22
|
dataset.data = new_data.to_a.collect{|r| r.to_csv }.join
|
23
23
|
dataset.save!
|
24
24
|
|
25
|
-
new_row = dataset.
|
26
|
-
new_row_month = dataset.
|
25
|
+
new_row = dataset.data.scoped[0]
|
26
|
+
new_row_month = dataset.data.scoped.collapse(:monthly)[0]
|
27
27
|
|
28
28
|
new_row[0].should_not eq new_row_month[0]
|
29
29
|
new_row[1].should eq new_row_month[1]
|
@@ -13,12 +13,35 @@ describe Quandl::Cassandra::Dataset do
|
|
13
13
|
its(:id){ should eq id }
|
14
14
|
its(:changes){ should eq( { "id" => [nil, id] }) }
|
15
15
|
|
16
|
+
describe ".new" do
|
17
|
+
let(:dataset){ Quandl::Cassandra::Dataset.new( id: 0 ) }
|
18
|
+
subject{ dataset }
|
19
|
+
its(:data){ should eq [] }
|
20
|
+
its(:column_ids){ should eq [] }
|
21
|
+
its(:columns){ should eq [] }
|
22
|
+
describe "#data" do
|
23
|
+
subject{ dataset.data }
|
24
|
+
its(:count){ should eq 0 }
|
25
|
+
its(:to_a){ should eq Quandl::Cassandra::Data.new }
|
26
|
+
context "given filters" do
|
27
|
+
subject{ dataset.data.row(0).column(1) }
|
28
|
+
its(:to_a){ should eq Quandl::Cassandra::Data.new }
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe "#save" do
|
34
|
+
subject{ Quandl::Cassandra::Dataset.new }
|
35
|
+
before(:each){ subject.save }
|
36
|
+
its(:new_record?){ should be_true }
|
37
|
+
end
|
38
|
+
|
16
39
|
describe "#class" do
|
17
40
|
subject{ dataset.class }
|
18
41
|
its(:table_name){ should eq 'datasets' }
|
19
42
|
end
|
20
43
|
|
21
|
-
|
44
|
+
describe "#save" do
|
22
45
|
|
23
46
|
before(:each){
|
24
47
|
dataset.data = Quandl::Fabricate::Data.rand(rows: 10, columns: 2, nils: false)
|
@@ -31,7 +54,11 @@ describe Quandl::Cassandra::Dataset do
|
|
31
54
|
|
32
55
|
describe ".find" do
|
33
56
|
subject{ Quandl::Cassandra::Dataset.find(id) }
|
34
|
-
its(:data){ should eq dataset.data }
|
57
|
+
its(:data){ should eq dataset.data.to_table }
|
58
|
+
it "data should count and return data" do
|
59
|
+
subject.data.count.should eq 10
|
60
|
+
subject.data.to_table.should be_a Quandl::Cassandra::Data
|
61
|
+
end
|
35
62
|
it "columns should eq dataset.columns" do
|
36
63
|
subject.columns.collect{|c| c.id.to_s }.should eq dataset.columns.collect{|c| c.id.to_s }
|
37
64
|
end
|
@@ -40,6 +67,19 @@ describe Quandl::Cassandra::Dataset do
|
|
40
67
|
end
|
41
68
|
end
|
42
69
|
|
70
|
+
describe "#reload" do
|
71
|
+
before(:each){
|
72
|
+
dataset.data.limit(5).to_a
|
73
|
+
dataset.reload
|
74
|
+
}
|
75
|
+
|
76
|
+
describe "#attributes" do
|
77
|
+
subject{ dataset.attributes }
|
78
|
+
its([:data]){ should eq nil }
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
|
43
83
|
describe "#data" do
|
44
84
|
subject{ dataset.data }
|
45
85
|
its(:count){ should eq 10 }
|
@@ -57,7 +97,6 @@ describe Quandl::Cassandra::Dataset do
|
|
57
97
|
its(:first){ should be_a Quandl::Cassandra::ColumnAttribute }
|
58
98
|
end
|
59
99
|
|
60
|
-
|
61
100
|
end
|
62
101
|
|
63
102
|
end
|
@@ -8,7 +8,8 @@ describe Multiset do
|
|
8
8
|
let(:d1){ create(:dataset) }
|
9
9
|
let(:d2){ create(:dataset) }
|
10
10
|
subject { Multiset.with_columns("#{d2.id.to_i}.1,#{d1.id.to_i}.1,#{d2.id.to_i}.3,#{d1.id.to_i}.2") }
|
11
|
-
|
11
|
+
|
12
|
+
let(:data_table){ subject.data.scoped.to_table }
|
12
13
|
|
13
14
|
it "should change the scope" do
|
14
15
|
scope = subject.data.transform('rdiff')
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: quandl_cassandra
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-10-
|
12
|
+
date: 2013-10-31 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -130,7 +130,7 @@ dependencies:
|
|
130
130
|
requirements:
|
131
131
|
- - ~>
|
132
132
|
- !ruby/object:Gem::Version
|
133
|
-
version: '0.
|
133
|
+
version: '0.4'
|
134
134
|
type: :runtime
|
135
135
|
prerelease: false
|
136
136
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -138,7 +138,7 @@ dependencies:
|
|
138
138
|
requirements:
|
139
139
|
- - ~>
|
140
140
|
- !ruby/object:Gem::Version
|
141
|
-
version: '0.
|
141
|
+
version: '0.4'
|
142
142
|
- !ruby/object:Gem::Dependency
|
143
143
|
name: quandl_data
|
144
144
|
requirement: !ruby/object:Gem::Requirement
|
@@ -245,19 +245,20 @@ files:
|
|
245
245
|
- lib/quandl/cassandra_models/column/read.rb
|
246
246
|
- lib/quandl/cassandra_models/column/read/collapse.rb
|
247
247
|
- lib/quandl/cassandra_models/column/read/column.rb
|
248
|
-
- lib/quandl/cassandra_models/column/read/
|
248
|
+
- lib/quandl/cassandra_models/column/read/data.rb
|
249
249
|
- lib/quandl/cassandra_models/column/read/offset.rb
|
250
|
-
- lib/quandl/cassandra_models/column/read/query.rb
|
251
250
|
- lib/quandl/cassandra_models/column/read/row.rb
|
251
|
+
- lib/quandl/cassandra_models/column/read/select_columns.rb
|
252
252
|
- lib/quandl/cassandra_models/column/read/transform.rb
|
253
253
|
- lib/quandl/cassandra_models/column/read/type.rb
|
254
254
|
- lib/quandl/cassandra_models/column/write.rb
|
255
255
|
- lib/quandl/cassandra_models/column/write/group_data_by_column.rb
|
256
256
|
- lib/quandl/cassandra_models/column/write/group_data_by_frequency.rb
|
257
|
+
- lib/quandl/cassandra_models/column/write/insert_column_attributes.rb
|
257
258
|
- lib/quandl/cassandra_models/column/write/insert_columns.rb
|
258
|
-
- lib/quandl/cassandra_models/column/write/insert_data.rb
|
259
259
|
- lib/quandl/cassandra_models/column_attribute.rb
|
260
260
|
- lib/quandl/cassandra_models/data.rb
|
261
|
+
- lib/quandl/cassandra_models/data/search.rb
|
261
262
|
- lib/quandl/cassandra_models/dataset.rb
|
262
263
|
- lib/quandl/cassandra_models/dataset_attribute.rb
|
263
264
|
- lib/quandl/cassandra_models/multiset.rb
|
@@ -266,6 +267,7 @@ files:
|
|
266
267
|
- spec/expectations/string.rb
|
267
268
|
- spec/expectations/time.rb
|
268
269
|
- spec/factories/dataset.rb
|
270
|
+
- spec/lib/quandl/cassandra/base/sanitization_spec.rb
|
269
271
|
- spec/lib/quandl/cassandra/base/scoping_spec.rb
|
270
272
|
- spec/lib/quandl/cassandra_models/column/write/group_data_by_frequency_spec.rb
|
271
273
|
- spec/lib/quandl/cassandra_models/column/write_spec.rb
|
@@ -316,6 +318,7 @@ test_files:
|
|
316
318
|
- spec/expectations/string.rb
|
317
319
|
- spec/expectations/time.rb
|
318
320
|
- spec/factories/dataset.rb
|
321
|
+
- spec/lib/quandl/cassandra/base/sanitization_spec.rb
|
319
322
|
- spec/lib/quandl/cassandra/base/scoping_spec.rb
|
320
323
|
- spec/lib/quandl/cassandra_models/column/write/group_data_by_frequency_spec.rb
|
321
324
|
- spec/lib/quandl/cassandra_models/column/write_spec.rb
|
@@ -1,39 +0,0 @@
|
|
1
|
-
class Quandl::Cassandra::Column::Write::InsertData < Quandl::Cassandra::Column::Write
|
2
|
-
|
3
|
-
def perform
|
4
|
-
insert_data_in_batches.collect(&:value)
|
5
|
-
end
|
6
|
-
|
7
|
-
def insert_data_in_batches
|
8
|
-
futures = []
|
9
|
-
statements = []
|
10
|
-
frequency_column_data.each do |frequency, column_data|
|
11
|
-
column_data.each do |column_id, rows|
|
12
|
-
rows.each do |time_value|
|
13
|
-
# collect statements
|
14
|
-
statements << statement( column_id, frequency, time_value[0], time_value[1] )
|
15
|
-
# after 30 statements are collected, execute a batch insert
|
16
|
-
if statements.count >= Quandl::Cassandra.configuration.batch_size
|
17
|
-
# collect the futures
|
18
|
-
futures << execute_async_batch(statements)
|
19
|
-
# clear statements
|
20
|
-
statements = []
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
# execute any remaining statements
|
26
|
-
futures << execute_async_batch(statements) if statements.count > 0
|
27
|
-
futures
|
28
|
-
end
|
29
|
-
|
30
|
-
def execute_async_batch(statements)
|
31
|
-
batch = %Q{BEGIN UNLOGGED BATCH\n#{statements.join("\n")}\nAPPLY BATCH;}
|
32
|
-
future = Quandl::Cassandra::Base.execute_async( batch )
|
33
|
-
end
|
34
|
-
|
35
|
-
def statement( id, type, time, value )
|
36
|
-
"INSERT INTO columns (id, type, time, value) VALUES (#{id}, '#{type}', #{time}, #{value})"
|
37
|
-
end
|
38
|
-
|
39
|
-
end
|