quandl_client 2.10.2 → 2.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/UPGRADE.md +3 -19
- data/lib/quandl/client/base.rb +1 -1
- data/lib/quandl/client/base/attributes.rb +12 -12
- data/lib/quandl/client/base/benchmark.rb +39 -39
- data/lib/quandl/client/base/model.rb +37 -37
- data/lib/quandl/client/base/search.rb +67 -68
- data/lib/quandl/client/base/validation.rb +96 -96
- data/lib/quandl/client/middleware.rb +4 -4
- data/lib/quandl/client/middleware/parse_json.rb +77 -79
- data/lib/quandl/client/models/dataset/benchmark.rb +7 -8
- data/lib/quandl/client/models/dataset/validations.rb +121 -121
- data/lib/quandl/client/models/location.rb +5 -7
- data/lib/quandl/client/models/report.rb +9 -11
- data/lib/quandl/client/models/scraper.rb +15 -18
- data/lib/quandl/client/models/sheet.rb +46 -48
- data/lib/quandl/client/models/source.rb +43 -45
- data/lib/quandl/client/version.rb +11 -11
- data/lib/quandl/pattern.rb +32 -32
- data/lib/quandl/pattern/client.rb +7 -7
- data/quandl_client.gemspec +1 -1
- data/spec/config/quandl.rb +4 -4
- data/spec/lib/quandl/client/base_spec.rb +8 -0
- data/spec/lib/quandl/client/dataset_spec.rb +4 -4
- metadata +41 -74
- data/VERSION +0 -1
@@ -1,88 +1,86 @@
|
|
1
1
|
require 'json'
|
2
2
|
|
3
3
|
module Quandl
|
4
|
-
module Client
|
5
|
-
module Middleware
|
4
|
+
module Client
|
5
|
+
module Middleware
|
6
|
+
class ParseJSON < Faraday::Response::Middleware
|
6
7
|
|
7
|
-
|
8
|
+
def on_complete(env)
|
9
|
+
env[:body] = case env[:status]
|
10
|
+
when 204
|
11
|
+
parse('{}', env)
|
12
|
+
else
|
13
|
+
parse(env[:body], env)
|
14
|
+
end
|
15
|
+
end
|
8
16
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
else
|
14
|
-
parse(env[:body], env)
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def parse(body, env)
|
19
|
-
json = parse_json(body, env)
|
20
|
-
json.has_key?(:docs) ? format_collection( json, env ) : format_record( json, env )
|
21
|
-
end
|
22
|
-
|
23
|
-
def format_record(json, env)
|
24
|
-
errors = json.delete(:errors) || {}
|
25
|
-
metadata = json.delete(:metadata) || {}
|
26
|
-
# collect some response data
|
27
|
-
metadata.merge!({
|
28
|
-
status: env[:status],
|
29
|
-
headers: env[:response_headers],
|
30
|
-
method: env[:method],
|
31
|
-
})
|
32
|
-
# return object
|
33
|
-
object = {
|
34
|
-
:data => json,
|
35
|
-
:errors => errors,
|
36
|
-
:metadata => metadata
|
37
|
-
}
|
38
|
-
env[:status] = 200
|
39
|
-
object
|
40
|
-
end
|
41
|
-
|
42
|
-
def format_collection(json, env)
|
43
|
-
errors = json.delete(:errors) || {}
|
44
|
-
metadata = json.delete(:metadata) || {}
|
45
|
-
docs = json.delete(:docs)
|
46
|
-
# collect some response data
|
47
|
-
metadata.merge!(json).merge!({
|
48
|
-
status: env[:status],
|
49
|
-
headers: env[:response_headers],
|
50
|
-
method: env[:method],
|
51
|
-
})
|
52
|
-
# each doc metadata references metadata
|
53
|
-
docs.each{|d| d[:metadata] = metadata }
|
54
|
-
# return object
|
55
|
-
object = {
|
56
|
-
:data => docs,
|
57
|
-
:errors => errors,
|
58
|
-
:metadata => metadata
|
59
|
-
}
|
60
|
-
env[:status] = 200
|
61
|
-
object
|
62
|
-
end
|
17
|
+
def parse(body, env)
|
18
|
+
json = parse_json(body, env)
|
19
|
+
json.has_key?(:docs) ? format_collection( json, env ) : format_record( json, env )
|
20
|
+
end
|
63
21
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
end
|
22
|
+
def format_record(json, env)
|
23
|
+
errors = json.delete(:errors) || {}
|
24
|
+
metadata = json.delete(:metadata) || {}
|
25
|
+
# collect some response data
|
26
|
+
metadata.merge!({
|
27
|
+
status: env[:status],
|
28
|
+
headers: env[:response_headers],
|
29
|
+
method: env[:method],
|
30
|
+
})
|
31
|
+
# return object
|
32
|
+
object = {
|
33
|
+
:data => json,
|
34
|
+
:errors => errors,
|
35
|
+
:metadata => metadata
|
36
|
+
}
|
37
|
+
env[:status] = 200
|
38
|
+
object
|
39
|
+
end
|
83
40
|
|
84
|
-
|
41
|
+
def format_collection(json, env)
|
42
|
+
errors = json.delete(:errors) || {}
|
43
|
+
metadata = json.delete(:metadata) || {}
|
44
|
+
docs = json.delete(:docs)
|
45
|
+
# collect some response data
|
46
|
+
metadata.merge!(json).merge!({
|
47
|
+
status: env[:status],
|
48
|
+
headers: env[:response_headers],
|
49
|
+
method: env[:method],
|
50
|
+
})
|
51
|
+
# each doc metadata references metadata
|
52
|
+
docs.each{|d| d[:metadata] = metadata }
|
53
|
+
# return object
|
54
|
+
object = {
|
55
|
+
:data => docs,
|
56
|
+
:errors => errors,
|
57
|
+
:metadata => metadata
|
58
|
+
}
|
59
|
+
env[:status] = 200
|
60
|
+
object
|
61
|
+
end
|
85
62
|
|
63
|
+
def parse_json(body = nil, env)
|
64
|
+
body ||= '{}'
|
65
|
+
json = begin
|
66
|
+
JSON.parse(body).symbolize_keys!
|
67
|
+
rescue JSON::ParserError
|
68
|
+
nil
|
69
|
+
end
|
70
|
+
# invalid json body?
|
71
|
+
if json.blank?
|
72
|
+
# fallback to error message
|
73
|
+
json = {
|
74
|
+
id: 1,
|
75
|
+
errors: {
|
76
|
+
parse_errors: [ "Quandl::Client::ParseJSON error. status: #{env[:status]}, body: #{body.inspect}" ]
|
77
|
+
}
|
78
|
+
}
|
79
|
+
end
|
80
|
+
json
|
81
|
+
end
|
82
|
+
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
86
|
end
|
87
|
-
end
|
88
|
-
end
|
@@ -1,11 +1,10 @@
|
|
1
1
|
class Quandl::Client::Dataset
|
2
|
-
module Benchmark
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
2
|
+
module Benchmark
|
3
|
+
extend ActiveSupport::Concern
|
4
|
+
|
5
|
+
included do
|
6
|
+
benchmark :data_should_be_valid!, :source_code_should_exist!, :save_dataset_data, :enforce_required_formats, :valid?
|
7
|
+
end
|
8
|
+
|
8
9
|
end
|
9
|
-
|
10
|
-
end
|
11
10
|
end
|
@@ -1,133 +1,133 @@
|
|
1
1
|
class Quandl::Client::Dataset
|
2
|
-
module Validations
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
end
|
23
|
-
|
24
|
-
def data_should_be_valid!
|
25
|
-
if data? && !data.valid?
|
26
|
-
data.errors.each{|k,v| self.errors.add( k,v ) }
|
27
|
-
return false
|
2
|
+
module Validations
|
3
|
+
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
included do
|
7
|
+
|
8
|
+
validates :code, presence: true, format: { with: Quandl::Pattern.code, message: "is invalid. Expected format: #{Quandl::Pattern.code.to_example}" }
|
9
|
+
validates :display_url, allow_blank: true, url: true
|
10
|
+
validate :data_should_be_valid!
|
11
|
+
validate :dataset_data_should_be_valid!
|
12
|
+
validate :data_row_count_should_match_column_count!
|
13
|
+
validate :data_columns_should_not_exceed_column_names!
|
14
|
+
validate :data_rows_should_have_equal_columns!
|
15
|
+
validate :ambiguous_code_requires_source_code!
|
16
|
+
|
17
|
+
validate :source_code_should_exist!
|
18
|
+
|
19
|
+
before_save :enforce_required_formats
|
20
|
+
after_save :save_dataset_data
|
21
|
+
|
28
22
|
end
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
23
|
+
|
24
|
+
def data_should_be_valid!
|
25
|
+
if data? && !data.valid?
|
26
|
+
data.errors.each{|k,v| self.errors.add( k,v ) }
|
27
|
+
return false
|
28
|
+
end
|
29
|
+
true
|
36
30
|
end
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
self.errors.add( :source_code, "Could not find a source with the source_code '#{source_code}'" ) if source.blank? || source.code.blank?
|
45
|
-
return false
|
31
|
+
|
32
|
+
def dataset_data_should_be_valid!
|
33
|
+
if dataset_data? && !dataset_data.valid?
|
34
|
+
dataset_data.errors.each{|k,v| self.errors.add( k,v ) }
|
35
|
+
return false
|
36
|
+
end
|
37
|
+
true
|
46
38
|
end
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
39
|
+
|
40
|
+
def source_code_should_exist!
|
41
|
+
if source_code.present?
|
42
|
+
Quandl::Client::Source.cached[source_code] = Quandl::Client::Source.find(source_code) unless Quandl::Client::Source.cached.has_key?(source_code)
|
43
|
+
source = Quandl::Client::Source.cached[source_code]
|
44
|
+
self.errors.add( :source_code, "Could not find a source with the source_code '#{source_code}'" ) if source.blank? || source.code.blank?
|
45
|
+
return false
|
46
|
+
end
|
47
|
+
true
|
55
48
|
end
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
49
|
+
|
50
|
+
def ambiguous_code_requires_source_code!
|
51
|
+
if code.to_s.numeric? && source_code.blank?
|
52
|
+
message = %Q{Pure numerical codes like "#{code}" are not allowed unless you include a source code. Do this:\nsource_code: <USERNAME>\ncode: #{code}}
|
53
|
+
self.errors.add( :data, message )
|
54
|
+
return false
|
55
|
+
end
|
56
|
+
true
|
63
57
|
end
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
column_count = data[0].count
|
72
|
-
# check each row
|
73
|
-
data.each_with_index do |row, index|
|
74
|
-
# the row is valid if it's count matches the first row's count
|
75
|
-
next if row.count == column_count
|
76
|
-
# the row is invalid if the count is mismatched
|
77
|
-
self.errors.add( :data, "Unexpected number of points in this row:\n#{row.join(',')}\nFound #{row.size-1} but expected #{data[0].size-1} based on precedent from the first row (#{data[0].join(',')})" )
|
78
|
-
# return validation failure
|
79
|
-
return false
|
58
|
+
|
59
|
+
def data_columns_should_not_exceed_column_names!
|
60
|
+
if errors.size == 0 && data? && data.present? && column_names.present? && data.first.count != column_names.count
|
61
|
+
self.errors.add( :data, "You may not change the number of columns in a dataset. This dataset has #{column_names.count} columns but you tried to send #{data.first.count} columns." )
|
62
|
+
return false
|
63
|
+
end
|
64
|
+
true
|
80
65
|
end
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
66
|
+
|
67
|
+
def data_rows_should_have_equal_columns!
|
68
|
+
# skip validation unless data is present
|
69
|
+
return true unless data? && data.present?
|
70
|
+
# use first row as expected column count
|
71
|
+
column_count = data[0].count
|
72
|
+
# check each row
|
73
|
+
data.each_with_index do |row, index|
|
74
|
+
# the row is valid if it's count matches the first row's count
|
75
|
+
next if row.count == column_count
|
76
|
+
# the row is invalid if the count is mismatched
|
77
|
+
self.errors.add( :data, "Unexpected number of points in this row:\n#{row.join(',')}\nFound #{row.size-1} but expected #{data[0].size-1} based on precedent from the first row (#{data[0].join(',')})" )
|
78
|
+
# return validation failure
|
79
|
+
return false
|
80
|
+
end
|
81
|
+
true
|
97
82
|
end
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
end
|
113
|
-
|
114
|
-
|
115
|
-
def inherit_errors(object)
|
116
|
-
return unless object.respond_to?(:response_errors) && object.response_errors.respond_to?(:each)
|
117
|
-
object.response_errors.each do |key, messages|
|
118
|
-
if messages.respond_to?(:each)
|
119
|
-
messages.each{|message| errors.add(key, message) }
|
83
|
+
|
84
|
+
def data_row_count_should_match_column_count!
|
85
|
+
# skip validation unless data and column_names present
|
86
|
+
return true unless data? && data.present? && column_names.present?
|
87
|
+
# count the number of expected columns
|
88
|
+
column_count = column_names.count
|
89
|
+
# check each row
|
90
|
+
data.each_with_index do |row, index|
|
91
|
+
# the row is valid if it's count matches the first row's count
|
92
|
+
next if row.count == column_count
|
93
|
+
# the row is invalid if the count is mismatched
|
94
|
+
self.errors.add( :data, "Unexpected number of points in this row:\n#{row.join(',')}\nFound #{row.size-1} but expected #{column_names.count-1} based on precedent from the header row (#{column_names.join(',')})" )
|
95
|
+
# return validation failure
|
96
|
+
return false
|
120
97
|
end
|
98
|
+
true
|
121
99
|
end
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
100
|
+
|
101
|
+
def save_dataset_data
|
102
|
+
return if (!saved? && id.blank?)
|
103
|
+
return if !data? || data.blank?
|
104
|
+
dataset_data.id = id
|
105
|
+
dataset_data.data = benchmark('data.to_csv'){ data.to_csv }
|
106
|
+
benchmark('dataset_data.save'){ dataset_data.save }
|
107
|
+
# update dataset's attributes with dataset_data's attributes
|
108
|
+
attributes.each{|k,v| attributes[k] = dataset_data.attributes[k] if dataset_data.attributes.has_key?(k) }
|
109
|
+
# update dataset errors with dataset_data
|
110
|
+
@metadata[:status] = dataset_data.status unless dataset_data.saved?
|
111
|
+
# inherit_errors(dataset_data) unless dataset_data.saved?
|
112
|
+
end
|
113
|
+
|
114
|
+
|
115
|
+
def inherit_errors(object)
|
116
|
+
return unless object.respond_to?(:response_errors) && object.response_errors.respond_to?(:each)
|
117
|
+
object.response_errors.each do |key, messages|
|
118
|
+
if messages.respond_to?(:each)
|
119
|
+
messages.each{|message| errors.add(key, message) }
|
120
|
+
end
|
121
|
+
end
|
122
|
+
@metadata[:status] = object.status
|
123
|
+
object
|
124
|
+
end
|
125
|
+
|
126
|
+
def enforce_required_formats
|
127
|
+
self.source_code = self.source_code.to_s.upcase
|
128
|
+
self.code = self.code.to_s.upcase
|
129
|
+
self.locations_attributes = locations_attributes.to_json if locations_attributes.respond_to?(:to_json) && !locations_attributes.kind_of?(String)
|
130
|
+
end
|
131
|
+
|
130
132
|
end
|
131
|
-
|
132
133
|
end
|
133
|
-
end
|