quandl_operation 0.1.21 → 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/UPGRADE.md +4 -0
- data/lib/quandl/operation/qdformat.rb +1 -1
- data/lib/quandl/operation/qdformat/dump.rb +8 -7
- data/lib/quandl/operation/qdformat/load.rb +23 -19
- data/lib/quandl/operation/qdformat/node.rb +5 -1
- data/lib/quandl/operation/version.rb +1 -1
- data/spec/lib/quandl/operation/qdformat/load_spec.rb +8 -6
- data/spec/lib/quandl/operation/qdformat/node_spec.rb +10 -7
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0ae58461349d4ba8c510ee5a0361b2502fb38867
|
4
|
+
data.tar.gz: a683b6d46cfa49cfb783fc8de468f22612f9e3e8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e83c29041365745773cffadbe0f913bcbbe828ed26bba60a71cd9b6a04dbdae620d4029e055ed74557e3bcbfcb37a150963d885cdd4cc97ba0fb338d6228d4c8
|
7
|
+
data.tar.gz: 61b2c18bdbcfd7efd021ae4e98029a2f890055f9794c3903cfaf425072622231131e0b88113723552f7e4f14af5986abe0f1267f9fe468efcd84078c12a8a91c
|
data/UPGRADE.md
CHANGED
@@ -4,6 +4,8 @@ class QDFormat
|
|
4
4
|
|
5
5
|
class Dump
|
6
6
|
|
7
|
+
ATTRIBUTES = [:source_code, :code, :name, :description]
|
8
|
+
|
7
9
|
class << self
|
8
10
|
|
9
11
|
def nodes(*args)
|
@@ -23,23 +25,22 @@ class Dump
|
|
23
25
|
end
|
24
26
|
|
25
27
|
def to_qdf
|
26
|
-
[attributes,
|
28
|
+
[ attributes,
|
29
|
+
"-\n",
|
30
|
+
column_names,
|
31
|
+
data
|
32
|
+
].compact.join
|
27
33
|
end
|
28
34
|
|
29
35
|
def attributes
|
30
|
-
attrs =
|
36
|
+
attrs = ATTRIBUTES.inject({}) do |memo, name|
|
31
37
|
name = name.to_s
|
32
38
|
memo[name] = node.send(name) if node.respond_to?(name) && node.send(name).present?
|
33
|
-
memo[name] = memo[name].gsub("\n", '\n') if memo[name].present?
|
34
39
|
memo
|
35
40
|
end
|
36
41
|
attrs.to_yaml[4..-1]
|
37
42
|
end
|
38
43
|
|
39
|
-
def description
|
40
|
-
record.description
|
41
|
-
end
|
42
|
-
|
43
44
|
def data
|
44
45
|
data = node.data.is_a?(Array) ? node.data.collect(&:to_csv).join : node.data
|
45
46
|
data = data.to_csv if data.respond_to?(:to_csv)
|
@@ -1,5 +1,7 @@
|
|
1
1
|
class Quandl::Operation::QDFormat::Load
|
2
2
|
|
3
|
+
SECTION_DELIMITER = '-'
|
4
|
+
|
3
5
|
class << self
|
4
6
|
|
5
7
|
def from_file(path)
|
@@ -7,22 +9,32 @@ class Quandl::Operation::QDFormat::Load
|
|
7
9
|
end
|
8
10
|
|
9
11
|
def from_string(input)
|
10
|
-
nodes = [
|
11
|
-
|
12
|
+
nodes = []
|
13
|
+
section_type = :data
|
14
|
+
input.each_line do |rline|
|
12
15
|
# strip whitespace
|
13
|
-
line =
|
16
|
+
line = rline.strip.rstrip
|
14
17
|
# ignore comments and blank lines
|
15
18
|
next if line[0] == '#' || line.blank?
|
16
|
-
|
17
|
-
|
18
|
-
# attribute_format denotes an attribute
|
19
|
+
|
20
|
+
# are we looking at an attribute?
|
19
21
|
if line =~ attribute_format
|
20
|
-
#
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
22
|
+
# if we are leaving the data section
|
23
|
+
# then this is the start of a new node
|
24
|
+
nodes << { attributes: '', data: '' } if section_type == :data
|
25
|
+
# update the section to attributes
|
26
|
+
section_type = :attributes
|
27
|
+
|
28
|
+
# have we reached the end of the attributes?
|
29
|
+
elsif line[0] == '-'
|
30
|
+
# update the section to data
|
31
|
+
section_type = :data
|
32
|
+
# skip to the next line
|
33
|
+
next
|
25
34
|
end
|
35
|
+
# add the line to it's section in the current node.
|
36
|
+
# YAML must include whitespace
|
37
|
+
nodes[-1][section_type] += (section_type == :data) ? "#{line}\n" : rline
|
26
38
|
end
|
27
39
|
# append the current node
|
28
40
|
nodes = parse_nodes(nodes)
|
@@ -37,10 +49,6 @@ class Quandl::Operation::QDFormat::Load
|
|
37
49
|
nodes.collect do |node|
|
38
50
|
# parse attrs as yaml
|
39
51
|
node[:attributes] = YAML.load( node[:attributes] )
|
40
|
-
# replace new line characters
|
41
|
-
node[:attributes].each do |key, value|
|
42
|
-
node[:attributes][key].gsub!('\n', "\n") if node[:attributes][key].respond_to?(:gsub!)
|
43
|
-
end
|
44
52
|
# parse data as csv
|
45
53
|
node[:attributes][:data] = CSV.parse(node[:data])
|
46
54
|
node
|
@@ -53,10 +61,6 @@ class Quandl::Operation::QDFormat::Load
|
|
53
61
|
end
|
54
62
|
end
|
55
63
|
|
56
|
-
def node_delimiter
|
57
|
-
'---'
|
58
|
-
end
|
59
|
-
|
60
64
|
def attribute_format
|
61
65
|
/^([a-z0-9_]+): (.+)/
|
62
66
|
end
|
@@ -30,7 +30,11 @@ class Quandl::Operation::QDFormat::Node
|
|
30
30
|
def full_code
|
31
31
|
[source_code, code].join('/')
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
|
+
def description=(value)
|
35
|
+
@description = value.to_s.gsub('\n', "\n")
|
36
|
+
end
|
37
|
+
|
34
38
|
def data=(rows)
|
35
39
|
self.column_names = rows.shift unless rows.first.collect{|r| r.to_s.numeric? }.include?(true)
|
36
40
|
@data = rows
|
@@ -6,22 +6,24 @@ describe Quandl::Operation::QDFormat::Load do
|
|
6
6
|
|
7
7
|
let(:qdf_dataset){
|
8
8
|
%Q{
|
9
|
-
#
|
9
|
+
# first dataset
|
10
10
|
source_code: NSE
|
11
11
|
code: OIL
|
12
12
|
name: Oil India Limited
|
13
|
-
description:
|
14
|
-
|
13
|
+
description: |-
|
14
|
+
Here is a description with multiple lines.
|
15
|
+
This is the second line.
|
16
|
+
-
|
15
17
|
Date, Value, High, Low
|
16
18
|
2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
|
17
19
|
2013-11-19,10.039388096885814,,14.09718770934256
|
18
|
-
---
|
19
20
|
|
21
|
+
# Second dataset
|
20
22
|
code: DATASET_CODE_2
|
21
23
|
source_code: SOURCE_CODE_2
|
22
24
|
name: Test Dataset Name 2
|
23
25
|
description: Here is a description with multiple lines.
|
24
|
-
|
26
|
+
-
|
25
27
|
Date, Value, High, Low
|
26
28
|
2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
|
27
29
|
2013-11-19,10.039388096885814,,14.09718770934256
|
@@ -43,7 +45,7 @@ describe Quandl::Operation::QDFormat::Load do
|
|
43
45
|
its(:source_code){ should eq 'NSE' }
|
44
46
|
its(:code){ should eq 'OIL' }
|
45
47
|
its(:name){ should eq 'Oil India Limited' }
|
46
|
-
its(:description){ should eq "
|
48
|
+
its(:description){ should eq "Here is a description with multiple lines.\nThis is the second line." }
|
47
49
|
its(:column_names){ should eq ['Date', 'Value', 'High', 'Low'] }
|
48
50
|
its(:data){ should eq [["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"],
|
49
51
|
["2013-11-19", "10.039388096885814", nil, "14.09718770934256"]] }
|
@@ -5,9 +5,9 @@ require 'spec_helper'
|
|
5
5
|
describe Quandl::Operation::QDFormat::Node do
|
6
6
|
|
7
7
|
let(:attributes) { {
|
8
|
-
code: '
|
8
|
+
code: 'DATASET_CODE_2',
|
9
9
|
source_code: 'SOURCE_CODE',
|
10
|
-
name: 'Test Dataset Name
|
10
|
+
name: 'Test Dataset Name 2',
|
11
11
|
description: "Here is a description with multiple lines.\n This is the second line.",
|
12
12
|
column_names: ['Date', 'Value', 'High', 'Low'],
|
13
13
|
data: [["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"],
|
@@ -16,9 +16,9 @@ describe Quandl::Operation::QDFormat::Node do
|
|
16
16
|
|
17
17
|
subject{ Quandl::Operation::QDFormat::Node.new(attributes) }
|
18
18
|
|
19
|
-
its(:code){ should eq '
|
19
|
+
its(:code){ should eq 'DATASET_CODE_2' }
|
20
20
|
its(:source_code){ should eq 'SOURCE_CODE' }
|
21
|
-
its(:name){ should eq 'Test Dataset Name
|
21
|
+
its(:name){ should eq 'Test Dataset Name 2' }
|
22
22
|
its(:description){ should eq "Here is a description with multiple lines.\n This is the second line." }
|
23
23
|
its(:column_names){ should eq ['Date', 'Value', 'High', 'Low'] }
|
24
24
|
its(:data){ should eq [["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"],
|
@@ -27,9 +27,12 @@ describe Quandl::Operation::QDFormat::Node do
|
|
27
27
|
its(:attributes){ should eq attributes }
|
28
28
|
|
29
29
|
its(:to_qdf){ should eq %Q{source_code: SOURCE_CODE
|
30
|
-
code:
|
31
|
-
name: Test Dataset Name
|
32
|
-
description:
|
30
|
+
code: DATASET_CODE_2
|
31
|
+
name: Test Dataset Name 2
|
32
|
+
description: |-
|
33
|
+
Here is a description with multiple lines.
|
34
|
+
This is the second line.
|
35
|
+
-
|
33
36
|
Date,Value,High,Low
|
34
37
|
2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
|
35
38
|
2013-11-19,10.039388096885814,,14.09718770934256
|