quandl_format 0.2.8 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +14 -0
- data/UPGRADE.md +7 -0
- data/VERSION +1 -0
- data/lib/quandl/format/dataset/attributes.rb +4 -4
- data/lib/quandl/format/dataset/client.rb +15 -12
- data/lib/quandl/format/dataset/load.rb +38 -23
- data/lib/quandl/format/version.rb +1 -1
- data/quandl_format.gemspec +2 -1
- data/spec/fixtures/data/illegal_dash.qdf +5 -0
- data/spec/fixtures/data/invalid_date.qdf +12 -0
- data/spec/lib/quandl/format/dataset/load/errors_spec.rb +44 -15
- data/spec/lib/quandl/format/dataset/load/validation_spec.rb +26 -0
- data/spec/lib/quandl/format/dataset/load_spec.rb +1 -1
- metadata +25 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d3736904b3e0d1c5ee9b90ebfaca235563775f56
|
4
|
+
data.tar.gz: f12745b06559794e379a999280cb4e6c28ea01ae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3537467133b0cb0f5b89732974bc034b2c5198970d2b215789a53d2a714ce63a8d09665b0d0afcd6a8b424404e07e7cdb199f34f3fe187ab8ec621ad4facc1d0
|
7
|
+
data.tar.gz: a4ad1d3c31abd11d3ee0a493f1e3126f28db56d6b7d124aec9825b48465b046ce3edabbd41aa596f71e88c89a51d8575f39a3695c49fdd80c423c3a17fbfbfc9
|
data/Rakefile
CHANGED
@@ -2,6 +2,8 @@ require "bundler"
|
|
2
2
|
require "rake"
|
3
3
|
require "bundler/gem_tasks"
|
4
4
|
require "rspec/core/rake_task"
|
5
|
+
require 'quandl/format'
|
6
|
+
require 'pry'
|
5
7
|
|
6
8
|
task :default => :spec
|
7
9
|
|
@@ -9,3 +11,15 @@ desc "Run all specs"
|
|
9
11
|
RSpec::Core::RakeTask.new(:spec) do |task|
|
10
12
|
task.pattern = "spec/**/*_spec.rb"
|
11
13
|
end
|
14
|
+
|
15
|
+
task :console do |t,args|
|
16
|
+
binding.pry
|
17
|
+
end
|
18
|
+
|
19
|
+
require 'quandl/utility/rake_tasks'
|
20
|
+
Quandl::Utility::Tasks.configure do |c|
|
21
|
+
c.name = 'quandl_format'
|
22
|
+
c.version_path = 'VERSION'
|
23
|
+
c.changelog_path = 'UPGRADE.md'
|
24
|
+
c.changelog_matching = ['^QUGC','^WIKI']
|
25
|
+
end
|
data/UPGRADE.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
## 0.3.0
|
2
|
+
|
3
|
+
* QUGC-104 bump gems to get onboard with new validations and error handling
|
4
|
+
* QUGC-104 Working towards a solution where all dataset validation happens in one place.
|
5
|
+
|
6
|
+
|
7
|
+
|
1
8
|
## 0.2.8
|
2
9
|
|
3
10
|
* QUGC-57 validation at an inappropriate time; move validations to Quandl::Client
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.3.0
|
@@ -60,11 +60,11 @@ module Attributes
|
|
60
60
|
|
61
61
|
def data=(rows)
|
62
62
|
rows = rows.to_table if rows.respond_to?(:to_table)
|
63
|
-
@data = Quandl::Data.new(rows)
|
63
|
+
@data = Quandl::Data.new(rows)
|
64
64
|
self.column_names = @data.headers if @data.headers.present?
|
65
65
|
@data
|
66
66
|
end
|
67
|
-
|
67
|
+
|
68
68
|
def column_names
|
69
69
|
@column_names ||= []
|
70
70
|
end
|
@@ -84,9 +84,9 @@ module Attributes
|
|
84
84
|
def attributes
|
85
85
|
self.class.attribute_names.inject({}){|m,k| m[k] = self.send(k) unless self.send(k).nil?; m }
|
86
86
|
end
|
87
|
-
|
87
|
+
|
88
88
|
private
|
89
|
-
|
89
|
+
|
90
90
|
def raise_unknown_attribute_error!(key)
|
91
91
|
m = "Unknown Field '#{key}' valid fields are: #{self.class.meta_attribute_names.join(', ')}"
|
92
92
|
raise Quandl::Error::UnknownAttribute, m
|
@@ -5,6 +5,13 @@ class Dataset
|
|
5
5
|
module Client
|
6
6
|
extend ActiveSupport::Concern
|
7
7
|
|
8
|
+
included do
|
9
|
+
include ActiveModel::Validations
|
10
|
+
|
11
|
+
validate :client_should_be_valid!
|
12
|
+
|
13
|
+
end
|
14
|
+
|
8
15
|
def human_errors
|
9
16
|
m = "#{client.human_status} \t #{client.full_url}"
|
10
17
|
return m if errors.blank?
|
@@ -28,15 +35,6 @@ module Client
|
|
28
35
|
client.save if valid?
|
29
36
|
end
|
30
37
|
|
31
|
-
def errors
|
32
|
-
client.error_messages
|
33
|
-
end
|
34
|
-
|
35
|
-
def valid?
|
36
|
-
assign_client_attributes
|
37
|
-
client.valid_with_server?
|
38
|
-
end
|
39
|
-
|
40
38
|
def client
|
41
39
|
@client ||= find_or_build_client
|
42
40
|
end
|
@@ -48,13 +46,18 @@ module Client
|
|
48
46
|
|
49
47
|
protected
|
50
48
|
|
51
|
-
def
|
52
|
-
client.
|
49
|
+
def client_should_be_valid!
|
50
|
+
if !client.valid_with_server?
|
51
|
+
client.errors.each{|err, value| self.errors.add( err, value ) }
|
52
|
+
return false
|
53
|
+
end
|
54
|
+
true
|
53
55
|
end
|
54
56
|
|
55
57
|
def find_or_build_client
|
56
|
-
@client
|
58
|
+
@client ||= Quandl::Client::Dataset.find(full_code)
|
57
59
|
@client = Quandl::Client::Dataset.new unless @client.try(:exists?)
|
60
|
+
@client.assign_attributes(attributes)
|
58
61
|
@client
|
59
62
|
end
|
60
63
|
|
@@ -89,15 +89,22 @@ class Quandl::Format::Dataset::Load
|
|
89
89
|
end
|
90
90
|
|
91
91
|
def process_node(node, &block)
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
92
|
+
begin
|
93
|
+
node = parse_node(node)
|
94
|
+
# fail on errored node
|
95
|
+
return false if node == false
|
96
|
+
# convert node to dataset
|
97
|
+
dataset = convert_node_to_dataset(node)
|
98
|
+
# do whatever we need to do with the node
|
99
|
+
block.call( dataset, nil ) unless dataset.nil?
|
100
|
+
# success
|
101
|
+
true
|
102
|
+
|
103
|
+
rescue Exception => err
|
104
|
+
block.call( nil, err )
|
105
|
+
false
|
106
|
+
|
107
|
+
end
|
101
108
|
end
|
102
109
|
|
103
110
|
def parse_node(node)
|
@@ -121,8 +128,8 @@ class Quandl::Format::Dataset::Load
|
|
121
128
|
end
|
122
129
|
attrs
|
123
130
|
rescue Exception => err
|
124
|
-
|
125
|
-
|
131
|
+
m = generate_yaml_parse_error(node, err)
|
132
|
+
raise err, m
|
126
133
|
end
|
127
134
|
|
128
135
|
def convert_node_to_dataset(node)
|
@@ -130,32 +137,40 @@ class Quandl::Format::Dataset::Load
|
|
130
137
|
dataset.data = node[:data]
|
131
138
|
dataset
|
132
139
|
rescue Exception => err
|
133
|
-
|
134
|
-
|
140
|
+
m = generate_dataset_error(node, err)
|
141
|
+
raise err, m
|
135
142
|
end
|
136
143
|
|
137
|
-
def
|
144
|
+
def generate_yaml_parse_error(node, err)
|
138
145
|
message = ""
|
139
146
|
if err.message == 'Unparsable input'
|
140
147
|
message = "Input data is unparsable. Are you missing a colon (:) or a space after a colon?\n"
|
141
|
-
elsif err.is_a?(Psych::SyntaxError)
|
148
|
+
elsif err.is_a?(Psych::SyntaxError) && err.respond_to?(:problem)
|
142
149
|
if err.problem =~ /mapping values are not allowed in this context/
|
143
|
-
message = "Syntax error before line #{1+node[:offset] + err.line}
|
150
|
+
message = "Syntax error *before* line #{1+node[:offset] + err.line}.\n"
|
151
|
+
if node[:attributes] =~ /:.+:/ # he probably has a colon in a field.
|
152
|
+
message += "You might have an illegal colon (:) in one of your fields. If so, use quotes.\n"
|
153
|
+
elsif node[:attributes] =~ /^([^:]+)$/ # he forgot the colon completely
|
154
|
+
message += "Did you forget a colon on this line:\n"
|
155
|
+
message += "#{$1}\n"
|
156
|
+
end
|
144
157
|
else
|
145
158
|
message += "Error parsing metadata. #{err.problem.capitalize} on line #{node[:offset] + err.line}\n"
|
146
159
|
if err.problem =~ /expected ':'/
|
147
160
|
message += "Did you forget to delimit the meta data section from the data section with a one or more dashes ('#{SYNTAX[:data]}')?\n"
|
148
161
|
end
|
149
162
|
end
|
163
|
+
elsif err.is_a?(Psych::SyntaxError)
|
164
|
+
message = err.to_s + "\n" + node[:attributes]
|
150
165
|
else
|
151
166
|
message += "Attribute parse error at line #{ node[:line] + err.line } column #{err.column}. #{err.problem} (#{err.class})\n" if node.has_key?(:line) && err.respond_to?(:line)
|
152
|
-
message += "Encountered error while parsing: \n " + node[:attributes].split("\n")[err.line - 1].to_s
|
167
|
+
message += "Encountered error while parsing: \n " + node[:attributes].split("\n")[err.line - 1].to_s if err.respond_to?(:line)
|
153
168
|
end
|
154
|
-
message += "
|
155
|
-
|
169
|
+
message += "\n"
|
170
|
+
message
|
156
171
|
end
|
157
172
|
|
158
|
-
def
|
173
|
+
def generate_dataset_error( node, err )
|
159
174
|
message = ''
|
160
175
|
message += node[:attributes][:source_code] + '/' if node[:attributes][:source_code].present?
|
161
176
|
message += node[:attributes][:code] + ' '
|
@@ -166,9 +181,9 @@ class Quandl::Format::Dataset::Load
|
|
166
181
|
message += "error around line #{node[:line]}\n"
|
167
182
|
end
|
168
183
|
# include original error
|
169
|
-
message += "#{$!} (#{err.class})
|
170
|
-
message += "
|
171
|
-
|
184
|
+
message += "#{$!} (#{err.class})"
|
185
|
+
message += "\n"
|
186
|
+
message
|
172
187
|
end
|
173
188
|
|
174
189
|
end
|
data/quandl_format.gemspec
CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
|
|
17
17
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
18
|
s.require_paths = ["lib"]
|
19
19
|
|
20
|
-
s.add_runtime_dependency "quandl_client", "~> 2.
|
20
|
+
s.add_runtime_dependency "quandl_client", "~> 2.6"
|
21
21
|
|
22
22
|
s.add_development_dependency "rake", "~> 10.0"
|
23
23
|
s.add_development_dependency "rspec", "~> 2.13"
|
@@ -26,4 +26,5 @@ Gem::Specification.new do |s|
|
|
26
26
|
s.add_development_dependency "simplecov"
|
27
27
|
s.add_development_dependency "guard"
|
28
28
|
s.add_development_dependency "guard-rspec"
|
29
|
+
s.add_development_dependency "quandl_utility"
|
29
30
|
end
|
@@ -2,21 +2,50 @@
|
|
2
2
|
require 'spec_helper'
|
3
3
|
|
4
4
|
describe Quandl::Format::Dataset do
|
5
|
+
|
6
|
+
let(:file_path){ 'spec/fixtures/data/' }
|
7
|
+
let(:file){ File.open( File.join(file_path, self.class.superclass.description + '.qdf')) }
|
8
|
+
let(:output){
|
9
|
+
output = []
|
10
|
+
Quandl::Format::Dataset.each_line( file ){|r,e| output << OpenStruct.new( record: r, error: e ) }
|
11
|
+
output
|
12
|
+
}
|
13
|
+
subject{ output.first }
|
14
|
+
|
15
|
+
context "unknown_attribute" do
|
16
|
+
its(:record){ should be_nil }
|
17
|
+
its("error.to_s"){ should match /this_attribute_does_not_exist/ }
|
18
|
+
end
|
5
19
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
Quandl::Format::Dataset.load( fixtures_data[file] )
|
10
|
-
end
|
20
|
+
context "invalid_yaml" do
|
21
|
+
its(:record){ should be_nil }
|
22
|
+
its("error.to_s"){ should match /Could not find expected ':'/ }
|
11
23
|
end
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
24
|
+
|
25
|
+
context "illegal_dash" do
|
26
|
+
its(:record){ should be_nil }
|
27
|
+
its("error.to_s"){ should match /Could not find expected ':'/ }
|
28
|
+
end
|
29
|
+
|
30
|
+
context "missing_dashes" do
|
31
|
+
subject{ output[2] }
|
32
|
+
its(:record){ should be_nil }
|
33
|
+
its("error.to_s"){ should match /Could not find expected ':' on line 22/ }
|
34
|
+
end
|
35
|
+
|
36
|
+
context "missing_colon" do
|
37
|
+
its(:record){ should be_nil }
|
38
|
+
its("error.to_s"){ should match /Did you forget a colon on this line/ }
|
39
|
+
end
|
40
|
+
|
41
|
+
context "missing_colon2" do
|
42
|
+
its(:record){ should be_nil }
|
43
|
+
its("error.to_s"){ should match /Could not find expected ':' on line 3/ }
|
44
|
+
end
|
45
|
+
|
46
|
+
context "missing_space" do
|
47
|
+
its(:record){ should be_nil }
|
48
|
+
its("error.to_s"){ should match /Are you missing a colon/ }
|
49
|
+
end
|
50
|
+
|
22
51
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'spec_helper'
|
3
|
+
|
4
|
+
describe Quandl::Format::Dataset do
|
5
|
+
|
6
|
+
let(:file){ self.class.superclass.description }
|
7
|
+
subject{ Quandl::Format::Dataset.load( fixtures_data[file] ).first }
|
8
|
+
|
9
|
+
context "invalid_data" do
|
10
|
+
before(:each){ subject.valid? }
|
11
|
+
its(:valid?){ should be_false }
|
12
|
+
its('errors.messages'){ should eq({ data: ["Invalid date segments. Expected yyyy-mm-dd received 'Date'"] }) }
|
13
|
+
end
|
14
|
+
|
15
|
+
context "invalid_date" do
|
16
|
+
before(:each){ subject.valid? }
|
17
|
+
its(:valid?){ should be_false }
|
18
|
+
|
19
|
+
its('errors.messages'){ should eq({ data: ["Invalid date 'ASDF'"] }) }
|
20
|
+
its('client.valid?'){ should be_false }
|
21
|
+
its('client.errors.messages'){ should eq({ data: ["Invalid date 'ASDF'"] }) }
|
22
|
+
its('client.data.valid?'){ should be_false }
|
23
|
+
its('client.data.errors.messages'){ should eq({ data: ["Invalid date 'ASDF'"] }) }
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: quandl_format
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Blake Hilscher
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-03-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: quandl_client
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ~>
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '2.
|
19
|
+
version: '2.6'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ~>
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '2.
|
26
|
+
version: '2.6'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -122,6 +122,20 @@ dependencies:
|
|
122
122
|
- - '>='
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: quandl_utility
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - '>='
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - '>='
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
125
139
|
description: Data will be loaded and dumped.
|
126
140
|
email:
|
127
141
|
- blake@hilscher.ca
|
@@ -136,6 +150,7 @@ files:
|
|
136
150
|
- README.md
|
137
151
|
- Rakefile
|
138
152
|
- UPGRADE.md
|
153
|
+
- VERSION
|
139
154
|
- examples/load.rb
|
140
155
|
- lib/quandl/client/dataset/to_qdf.rb
|
141
156
|
- lib/quandl/error/column_count_mismatch.rb
|
@@ -152,7 +167,9 @@ files:
|
|
152
167
|
- spec/config/logger.rb
|
153
168
|
- spec/fixtures/data/annual.qdf
|
154
169
|
- spec/fixtures/data/illegal_colon.qdf
|
170
|
+
- spec/fixtures/data/illegal_dash.qdf
|
155
171
|
- spec/fixtures/data/invalid_data.qdf
|
172
|
+
- spec/fixtures/data/invalid_date.qdf
|
156
173
|
- spec/fixtures/data/invalid_yaml.qdf
|
157
174
|
- spec/fixtures/data/metadata_only.qdf
|
158
175
|
- spec/fixtures/data/mismatched_columns.qdf
|
@@ -170,6 +187,7 @@ files:
|
|
170
187
|
- spec/lib/quandl/format/dataset/client_spec.rb
|
171
188
|
- spec/lib/quandl/format/dataset/load/errors_spec.rb
|
172
189
|
- spec/lib/quandl/format/dataset/load/valid_spec.rb
|
190
|
+
- spec/lib/quandl/format/dataset/load/validation_spec.rb
|
173
191
|
- spec/lib/quandl/format/dataset/load_spec.rb
|
174
192
|
- spec/lib/quandl/format/dataset_spec.rb
|
175
193
|
- spec/spec_helper.rb
|
@@ -202,7 +220,9 @@ test_files:
|
|
202
220
|
- spec/config/logger.rb
|
203
221
|
- spec/fixtures/data/annual.qdf
|
204
222
|
- spec/fixtures/data/illegal_colon.qdf
|
223
|
+
- spec/fixtures/data/illegal_dash.qdf
|
205
224
|
- spec/fixtures/data/invalid_data.qdf
|
225
|
+
- spec/fixtures/data/invalid_date.qdf
|
206
226
|
- spec/fixtures/data/invalid_yaml.qdf
|
207
227
|
- spec/fixtures/data/metadata_only.qdf
|
208
228
|
- spec/fixtures/data/mismatched_columns.qdf
|
@@ -220,6 +240,7 @@ test_files:
|
|
220
240
|
- spec/lib/quandl/format/dataset/client_spec.rb
|
221
241
|
- spec/lib/quandl/format/dataset/load/errors_spec.rb
|
222
242
|
- spec/lib/quandl/format/dataset/load/valid_spec.rb
|
243
|
+
- spec/lib/quandl/format/dataset/load/validation_spec.rb
|
223
244
|
- spec/lib/quandl/format/dataset/load_spec.rb
|
224
245
|
- spec/lib/quandl/format/dataset_spec.rb
|
225
246
|
- spec/spec_helper.rb
|