quandl_data 1.3.2 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/UPGRADE.md +6 -0
- data/lib/quandl/data.rb +0 -1
- data/lib/quandl/data/attributes.rb +1 -14
- data/lib/quandl/data/cleaning.rb +39 -4
- data/lib/quandl/data/format.rb +6 -0
- data/lib/quandl/data/version.rb +1 -1
- data/spec/lib/quandl/data_spec.rb +18 -0
- metadata +1 -4
- data/lib/quandl/babelfish/data.rb +0 -37
- data/spec/lib/quandl/babelfish/data_spec.rb +0 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b218225ec95c6b4ab6d1a6aca2e0123c723cbc05
|
4
|
+
data.tar.gz: 1b7650907d85479968f9e9ef5d53ff27476843df
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f11b51e91b3f270084abe1e7e4c017b0ac1261db46bd4f6d3e574c83552f40990741cd4de08010894960317c4f70edb59dfe0c000c8ed37601b099b4e3a58a9b
|
7
|
+
data.tar.gz: 8e3bf9e9cd563ab2e3a0798bf45ee240a75f8274be95c6d83c6839396121baca32584cdc84b01b7d05d8d18db6f50d01f103c1e3c743d3fc9ee37cf132f9d90d
|
data/UPGRADE.md
CHANGED
data/lib/quandl/data.rb
CHANGED
@@ -4,29 +4,16 @@ module Attributes
|
|
4
4
|
|
5
5
|
extend ActiveSupport::Concern
|
6
6
|
|
7
|
-
module ClassMethods
|
8
|
-
|
9
|
-
def with_type(type)
|
10
|
-
type = type.to_s.downcase.to_sym
|
11
|
-
case type
|
12
|
-
when :dirty, :babelfish then Quandl::Babelfish::Data
|
13
|
-
else
|
14
|
-
Quandl::Data
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
end
|
19
|
-
|
20
7
|
def initialize(*args)
|
21
8
|
self.pristine_data = args.first
|
22
9
|
self.attributes = args.extract_options!
|
23
10
|
end
|
24
11
|
|
25
12
|
def headers=(value)
|
26
|
-
# only assign arrays
|
27
13
|
@headers = value.flatten if value.kind_of?(Array)
|
28
14
|
end
|
29
15
|
def headers
|
16
|
+
ensure_data_is_cleaned
|
30
17
|
@headers
|
31
18
|
end
|
32
19
|
def headers?
|
data/lib/quandl/data/cleaning.rb
CHANGED
@@ -13,24 +13,59 @@ module Cleaning
|
|
13
13
|
def cleaned?
|
14
14
|
cleaned == true
|
15
15
|
end
|
16
|
+
|
17
|
+
def ensure_data_is_cleaned
|
18
|
+
data_array unless cleaned?
|
19
|
+
end
|
16
20
|
|
17
21
|
protected
|
18
22
|
|
19
23
|
def clean(data)
|
24
|
+
# check if data is dirty
|
25
|
+
requires_cleaning = ensure_data_requires_cleaning(data)
|
26
|
+
# short ciruit unless data is dirty
|
27
|
+
return requires_cleaning unless requires_cleaning == true
|
28
|
+
# ensure we're dealing with an array
|
29
|
+
data = ensure_data_is_an_array(data)
|
30
|
+
# clean with either format or babelfish
|
31
|
+
known_format?( data ) ? clean_with_format(data) : clean_with_babelfish(data)
|
32
|
+
end
|
33
|
+
|
34
|
+
def ensure_data_requires_cleaning(data)
|
20
35
|
# skip cleaning if already clean
|
21
36
|
return data if data.kind_of?(Array) && cleaned?
|
22
37
|
# Quandl::Data is already clean, but to avoid errors extract internal array
|
23
|
-
return data.
|
38
|
+
return data.to_date.to_a if data.kind_of?(Quandl::Data)
|
24
39
|
# Return empty array if given empty string, nil, etc.
|
25
40
|
return [] if data.blank?
|
41
|
+
# data requires cleaning
|
42
|
+
true
|
43
|
+
end
|
44
|
+
|
45
|
+
def ensure_data_is_an_array(data)
|
46
|
+
# Hash needs conversion to array
|
47
|
+
data = Quandl::Data::Format.hash_to_array( data )
|
26
48
|
# String needs conversion to array
|
49
|
+
data = Quandl::Data::Format.csv_to_array( data )
|
50
|
+
data
|
51
|
+
end
|
52
|
+
|
53
|
+
def known_format?( data )
|
54
|
+
Format.recognized_date?( data[0][0] )
|
55
|
+
end
|
56
|
+
|
57
|
+
def clean_with_format(data)
|
27
58
|
data = Format.parse( data )
|
28
|
-
# mark data as clean
|
29
59
|
cleaned!
|
30
|
-
# return data
|
31
60
|
data
|
32
61
|
end
|
33
|
-
|
62
|
+
|
63
|
+
def clean_with_babelfish(data)
|
64
|
+
data, self.headers = Quandl::Babelfish.clean(data)
|
65
|
+
cleaned!
|
66
|
+
data
|
67
|
+
end
|
68
|
+
|
34
69
|
def cleaned!
|
35
70
|
self.cleaned = true
|
36
71
|
end
|
data/lib/quandl/data/format.rb
CHANGED
@@ -57,6 +57,12 @@ class Format
|
|
57
57
|
data
|
58
58
|
end
|
59
59
|
|
60
|
+
def recognized_date?( value )
|
61
|
+
return true if value.is_a?(Date)
|
62
|
+
return true if value.is_a?(String) && value =~ /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/
|
63
|
+
false
|
64
|
+
end
|
65
|
+
|
60
66
|
def date_format?(data)
|
61
67
|
value = data[0][0]
|
62
68
|
# julian date?
|
data/lib/quandl/data/version.rb
CHANGED
@@ -41,4 +41,22 @@ describe Quandl::Data do
|
|
41
41
|
its("#{o1}!"){ should eq subject }
|
42
42
|
end
|
43
43
|
|
44
|
+
context "babelfish" do
|
45
|
+
|
46
|
+
let(:csv) { "Date, Column 1, Column 2, C3, C4, C5, C6, C7\n 2012-03-07,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-06,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-05,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-04,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-29,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-28,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n" }
|
47
|
+
|
48
|
+
let(:data){ Quandl::Data.new(csv) }
|
49
|
+
subject{ data }
|
50
|
+
|
51
|
+
let(:expected_data){ [['2012-03-07',nil,69.75,69.75,69.75,0.0,0.0,0.0],['2012-03-06',69.75,69.75,69.75,69.75,0.0,0.0,0.0],['2012-03-05',69.75,69.75,69.75,69.75,0.0,0.0,0.0],['2012-03-04',69.75,69.75,69.75,69.75,0.0,0.0,0.0],['2012-02-29',nil,69.75,69.75,69.75,0.0,0.0,0.0],['2012-02-28',69.75,69.75,69.75,69.75,0.0,0.0,0.0]] }
|
52
|
+
let(:expected_headers){ ['Date', 'Column 1', 'Column 2', 'C3', 'C4', 'C5', 'C6', 'C7'] }
|
53
|
+
|
54
|
+
its(:headers){ should eq expected_headers }
|
55
|
+
its('clone.headers'){ should eq expected_headers }
|
56
|
+
its('to_jd.headers'){ should eq expected_headers }
|
57
|
+
|
58
|
+
its(:to_date_str){ should eq expected_data }
|
59
|
+
|
60
|
+
end
|
61
|
+
|
44
62
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: quandl_data
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Blake Hilscher
|
@@ -153,7 +153,6 @@ files:
|
|
153
153
|
- README.md
|
154
154
|
- Rakefile
|
155
155
|
- UPGRADE.md
|
156
|
-
- lib/quandl/babelfish/data.rb
|
157
156
|
- lib/quandl/data.rb
|
158
157
|
- lib/quandl/data/attributes.rb
|
159
158
|
- lib/quandl/data/cleaning.rb
|
@@ -167,7 +166,6 @@ files:
|
|
167
166
|
- lib/quandl/fabricate/data.rb
|
168
167
|
- quandl_data.gemspec
|
169
168
|
- spec/lib/fabricate/data_spec.rb
|
170
|
-
- spec/lib/quandl/babelfish/data_spec.rb
|
171
169
|
- spec/lib/quandl/data/enumerator_spec.rb
|
172
170
|
- spec/lib/quandl/data/format_spec.rb
|
173
171
|
- spec/lib/quandl/data/operations_spec.rb
|
@@ -199,7 +197,6 @@ specification_version: 4
|
|
199
197
|
summary: For interfacing with data
|
200
198
|
test_files:
|
201
199
|
- spec/lib/fabricate/data_spec.rb
|
202
|
-
- spec/lib/quandl/babelfish/data_spec.rb
|
203
200
|
- spec/lib/quandl/data/enumerator_spec.rb
|
204
201
|
- spec/lib/quandl/data/format_spec.rb
|
205
202
|
- spec/lib/quandl/data/operations_spec.rb
|
@@ -1,37 +0,0 @@
|
|
1
|
-
module Quandl
|
2
|
-
module Babelfish
|
3
|
-
|
4
|
-
class Data < Quandl::Data
|
5
|
-
|
6
|
-
def initialize(*args)
|
7
|
-
super(*args)
|
8
|
-
# clean data on initialize
|
9
|
-
self.data_array
|
10
|
-
end
|
11
|
-
|
12
|
-
|
13
|
-
protected
|
14
|
-
|
15
|
-
def clean(data)
|
16
|
-
# skip cleaning if already clean
|
17
|
-
return data if data.kind_of?(Array) && cleaned?
|
18
|
-
# Quandl::Data is already clean, but to avoid errors extract internal array
|
19
|
-
return data.data_array if data.kind_of?(Quandl::Data)
|
20
|
-
# Return empty array if given empty string, nil, etc.
|
21
|
-
return [] if data.blank?
|
22
|
-
# Hash needs conversion to array
|
23
|
-
data = Quandl::Data::Format.hash_to_array( data )
|
24
|
-
# String needs conversion to array
|
25
|
-
data = Quandl::Data::Format.csv_to_array( data )
|
26
|
-
# Babelfish cleaner
|
27
|
-
data, self.headers = Quandl::Babelfish.clean(data)
|
28
|
-
# mark data as clean
|
29
|
-
cleaned!
|
30
|
-
# return data
|
31
|
-
data
|
32
|
-
end
|
33
|
-
|
34
|
-
end
|
35
|
-
|
36
|
-
end
|
37
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Quandl::Babelfish::Data do
|
4
|
-
|
5
|
-
let(:csv) { "Date, Column 1, Column 2, C3, C4, C5, C6, C7\n 2012-03-07,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-06,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-05,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-04,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-29,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-28,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n" }
|
6
|
-
|
7
|
-
let(:data){ Quandl::Babelfish::Data.new(csv) }
|
8
|
-
subject{ data }
|
9
|
-
|
10
|
-
let(:expected_data){ [['2012-03-07',nil,69.75,69.75,69.75,0.0,0.0,0.0],['2012-03-06',69.75,69.75,69.75,69.75,0.0,0.0,0.0],['2012-03-05',69.75,69.75,69.75,69.75,0.0,0.0,0.0],['2012-03-04',69.75,69.75,69.75,69.75,0.0,0.0,0.0],['2012-02-29',nil,69.75,69.75,69.75,0.0,0.0,0.0],['2012-02-28',69.75,69.75,69.75,69.75,0.0,0.0,0.0]] }
|
11
|
-
let(:expected_headers){ ['Date', 'Column 1', 'Column 2', 'C3', 'C4', 'C5', 'C6', 'C7'] }
|
12
|
-
|
13
|
-
its(:headers){ should eq expected_headers }
|
14
|
-
its('clone.headers'){ should eq expected_headers }
|
15
|
-
its('to_jd.headers'){ should eq expected_headers }
|
16
|
-
|
17
|
-
its(:to_date_str){ should eq expected_data }
|
18
|
-
|
19
|
-
end
|