quandl_client 2.10.2 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/UPGRADE.md +3 -19
- data/lib/quandl/client/base.rb +1 -1
- data/lib/quandl/client/base/attributes.rb +12 -12
- data/lib/quandl/client/base/benchmark.rb +39 -39
- data/lib/quandl/client/base/model.rb +37 -37
- data/lib/quandl/client/base/search.rb +67 -68
- data/lib/quandl/client/base/validation.rb +96 -96
- data/lib/quandl/client/middleware.rb +4 -4
- data/lib/quandl/client/middleware/parse_json.rb +77 -79
- data/lib/quandl/client/models/dataset/benchmark.rb +7 -8
- data/lib/quandl/client/models/dataset/validations.rb +121 -121
- data/lib/quandl/client/models/location.rb +5 -7
- data/lib/quandl/client/models/report.rb +9 -11
- data/lib/quandl/client/models/scraper.rb +15 -18
- data/lib/quandl/client/models/sheet.rb +46 -48
- data/lib/quandl/client/models/source.rb +43 -45
- data/lib/quandl/client/version.rb +11 -11
- data/lib/quandl/pattern.rb +32 -32
- data/lib/quandl/pattern/client.rb +7 -7
- data/quandl_client.gemspec +1 -1
- data/spec/config/quandl.rb +4 -4
- data/spec/lib/quandl/client/base_spec.rb +8 -0
- data/spec/lib/quandl/client/dataset_spec.rb +4 -4
- metadata +41 -74
- data/VERSION +0 -1
@@ -1,88 +1,86 @@
|
|
1
1
|
require 'json'
|
2
2
|
|
3
3
|
module Quandl
|
4
|
-
module Client
|
5
|
-
module Middleware
|
4
|
+
module Client
|
5
|
+
module Middleware
|
6
|
+
class ParseJSON < Faraday::Response::Middleware
|
6
7
|
|
7
|
-
|
8
|
+
def on_complete(env)
|
9
|
+
env[:body] = case env[:status]
|
10
|
+
when 204
|
11
|
+
parse('{}', env)
|
12
|
+
else
|
13
|
+
parse(env[:body], env)
|
14
|
+
end
|
15
|
+
end
|
8
16
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
else
|
14
|
-
parse(env[:body], env)
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def parse(body, env)
|
19
|
-
json = parse_json(body, env)
|
20
|
-
json.has_key?(:docs) ? format_collection( json, env ) : format_record( json, env )
|
21
|
-
end
|
22
|
-
|
23
|
-
def format_record(json, env)
|
24
|
-
errors = json.delete(:errors) || {}
|
25
|
-
metadata = json.delete(:metadata) || {}
|
26
|
-
# collect some response data
|
27
|
-
metadata.merge!({
|
28
|
-
status: env[:status],
|
29
|
-
headers: env[:response_headers],
|
30
|
-
method: env[:method],
|
31
|
-
})
|
32
|
-
# return object
|
33
|
-
object = {
|
34
|
-
:data => json,
|
35
|
-
:errors => errors,
|
36
|
-
:metadata => metadata
|
37
|
-
}
|
38
|
-
env[:status] = 200
|
39
|
-
object
|
40
|
-
end
|
41
|
-
|
42
|
-
def format_collection(json, env)
|
43
|
-
errors = json.delete(:errors) || {}
|
44
|
-
metadata = json.delete(:metadata) || {}
|
45
|
-
docs = json.delete(:docs)
|
46
|
-
# collect some response data
|
47
|
-
metadata.merge!(json).merge!({
|
48
|
-
status: env[:status],
|
49
|
-
headers: env[:response_headers],
|
50
|
-
method: env[:method],
|
51
|
-
})
|
52
|
-
# each doc metadata references metadata
|
53
|
-
docs.each{|d| d[:metadata] = metadata }
|
54
|
-
# return object
|
55
|
-
object = {
|
56
|
-
:data => docs,
|
57
|
-
:errors => errors,
|
58
|
-
:metadata => metadata
|
59
|
-
}
|
60
|
-
env[:status] = 200
|
61
|
-
object
|
62
|
-
end
|
17
|
+
def parse(body, env)
|
18
|
+
json = parse_json(body, env)
|
19
|
+
json.has_key?(:docs) ? format_collection( json, env ) : format_record( json, env )
|
20
|
+
end
|
63
21
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
end
|
22
|
+
def format_record(json, env)
|
23
|
+
errors = json.delete(:errors) || {}
|
24
|
+
metadata = json.delete(:metadata) || {}
|
25
|
+
# collect some response data
|
26
|
+
metadata.merge!({
|
27
|
+
status: env[:status],
|
28
|
+
headers: env[:response_headers],
|
29
|
+
method: env[:method],
|
30
|
+
})
|
31
|
+
# return object
|
32
|
+
object = {
|
33
|
+
:data => json,
|
34
|
+
:errors => errors,
|
35
|
+
:metadata => metadata
|
36
|
+
}
|
37
|
+
env[:status] = 200
|
38
|
+
object
|
39
|
+
end
|
83
40
|
|
84
|
-
|
41
|
+
def format_collection(json, env)
|
42
|
+
errors = json.delete(:errors) || {}
|
43
|
+
metadata = json.delete(:metadata) || {}
|
44
|
+
docs = json.delete(:docs)
|
45
|
+
# collect some response data
|
46
|
+
metadata.merge!(json).merge!({
|
47
|
+
status: env[:status],
|
48
|
+
headers: env[:response_headers],
|
49
|
+
method: env[:method],
|
50
|
+
})
|
51
|
+
# each doc metadata references metadata
|
52
|
+
docs.each{|d| d[:metadata] = metadata }
|
53
|
+
# return object
|
54
|
+
object = {
|
55
|
+
:data => docs,
|
56
|
+
:errors => errors,
|
57
|
+
:metadata => metadata
|
58
|
+
}
|
59
|
+
env[:status] = 200
|
60
|
+
object
|
61
|
+
end
|
85
62
|
|
63
|
+
def parse_json(body = nil, env)
|
64
|
+
body ||= '{}'
|
65
|
+
json = begin
|
66
|
+
JSON.parse(body).symbolize_keys!
|
67
|
+
rescue JSON::ParserError
|
68
|
+
nil
|
69
|
+
end
|
70
|
+
# invalid json body?
|
71
|
+
if json.blank?
|
72
|
+
# fallback to error message
|
73
|
+
json = {
|
74
|
+
id: 1,
|
75
|
+
errors: {
|
76
|
+
parse_errors: [ "Quandl::Client::ParseJSON error. status: #{env[:status]}, body: #{body.inspect}" ]
|
77
|
+
}
|
78
|
+
}
|
79
|
+
end
|
80
|
+
json
|
81
|
+
end
|
82
|
+
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
86
|
end
|
87
|
-
end
|
88
|
-
end
|
@@ -1,11 +1,10 @@
|
|
1
1
|
class Quandl::Client::Dataset
|
2
|
-
module Benchmark
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
2
|
+
module Benchmark
|
3
|
+
extend ActiveSupport::Concern
|
4
|
+
|
5
|
+
included do
|
6
|
+
benchmark :data_should_be_valid!, :source_code_should_exist!, :save_dataset_data, :enforce_required_formats, :valid?
|
7
|
+
end
|
8
|
+
|
8
9
|
end
|
9
|
-
|
10
|
-
end
|
11
10
|
end
|
@@ -1,133 +1,133 @@
|
|
1
1
|
class Quandl::Client::Dataset
|
2
|
-
module Validations
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
end
|
23
|
-
|
24
|
-
def data_should_be_valid!
|
25
|
-
if data? && !data.valid?
|
26
|
-
data.errors.each{|k,v| self.errors.add( k,v ) }
|
27
|
-
return false
|
2
|
+
module Validations
|
3
|
+
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
included do
|
7
|
+
|
8
|
+
validates :code, presence: true, format: { with: Quandl::Pattern.code, message: "is invalid. Expected format: #{Quandl::Pattern.code.to_example}" }
|
9
|
+
validates :display_url, allow_blank: true, url: true
|
10
|
+
validate :data_should_be_valid!
|
11
|
+
validate :dataset_data_should_be_valid!
|
12
|
+
validate :data_row_count_should_match_column_count!
|
13
|
+
validate :data_columns_should_not_exceed_column_names!
|
14
|
+
validate :data_rows_should_have_equal_columns!
|
15
|
+
validate :ambiguous_code_requires_source_code!
|
16
|
+
|
17
|
+
validate :source_code_should_exist!
|
18
|
+
|
19
|
+
before_save :enforce_required_formats
|
20
|
+
after_save :save_dataset_data
|
21
|
+
|
28
22
|
end
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
23
|
+
|
24
|
+
def data_should_be_valid!
|
25
|
+
if data? && !data.valid?
|
26
|
+
data.errors.each{|k,v| self.errors.add( k,v ) }
|
27
|
+
return false
|
28
|
+
end
|
29
|
+
true
|
36
30
|
end
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
self.errors.add( :source_code, "Could not find a source with the source_code '#{source_code}'" ) if source.blank? || source.code.blank?
|
45
|
-
return false
|
31
|
+
|
32
|
+
def dataset_data_should_be_valid!
|
33
|
+
if dataset_data? && !dataset_data.valid?
|
34
|
+
dataset_data.errors.each{|k,v| self.errors.add( k,v ) }
|
35
|
+
return false
|
36
|
+
end
|
37
|
+
true
|
46
38
|
end
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
39
|
+
|
40
|
+
def source_code_should_exist!
|
41
|
+
if source_code.present?
|
42
|
+
Quandl::Client::Source.cached[source_code] = Quandl::Client::Source.find(source_code) unless Quandl::Client::Source.cached.has_key?(source_code)
|
43
|
+
source = Quandl::Client::Source.cached[source_code]
|
44
|
+
self.errors.add( :source_code, "Could not find a source with the source_code '#{source_code}'" ) if source.blank? || source.code.blank?
|
45
|
+
return false
|
46
|
+
end
|
47
|
+
true
|
55
48
|
end
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
49
|
+
|
50
|
+
def ambiguous_code_requires_source_code!
|
51
|
+
if code.to_s.numeric? && source_code.blank?
|
52
|
+
message = %Q{Pure numerical codes like "#{code}" are not allowed unless you include a source code. Do this:\nsource_code: <USERNAME>\ncode: #{code}}
|
53
|
+
self.errors.add( :data, message )
|
54
|
+
return false
|
55
|
+
end
|
56
|
+
true
|
63
57
|
end
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
column_count = data[0].count
|
72
|
-
# check each row
|
73
|
-
data.each_with_index do |row, index|
|
74
|
-
# the row is valid if it's count matches the first row's count
|
75
|
-
next if row.count == column_count
|
76
|
-
# the row is invalid if the count is mismatched
|
77
|
-
self.errors.add( :data, "Unexpected number of points in this row:\n#{row.join(',')}\nFound #{row.size-1} but expected #{data[0].size-1} based on precedent from the first row (#{data[0].join(',')})" )
|
78
|
-
# return validation failure
|
79
|
-
return false
|
58
|
+
|
59
|
+
def data_columns_should_not_exceed_column_names!
|
60
|
+
if errors.size == 0 && data? && data.present? && column_names.present? && data.first.count != column_names.count
|
61
|
+
self.errors.add( :data, "You may not change the number of columns in a dataset. This dataset has #{column_names.count} columns but you tried to send #{data.first.count} columns." )
|
62
|
+
return false
|
63
|
+
end
|
64
|
+
true
|
80
65
|
end
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
66
|
+
|
67
|
+
def data_rows_should_have_equal_columns!
|
68
|
+
# skip validation unless data is present
|
69
|
+
return true unless data? && data.present?
|
70
|
+
# use first row as expected column count
|
71
|
+
column_count = data[0].count
|
72
|
+
# check each row
|
73
|
+
data.each_with_index do |row, index|
|
74
|
+
# the row is valid if it's count matches the first row's count
|
75
|
+
next if row.count == column_count
|
76
|
+
# the row is invalid if the count is mismatched
|
77
|
+
self.errors.add( :data, "Unexpected number of points in this row:\n#{row.join(',')}\nFound #{row.size-1} but expected #{data[0].size-1} based on precedent from the first row (#{data[0].join(',')})" )
|
78
|
+
# return validation failure
|
79
|
+
return false
|
80
|
+
end
|
81
|
+
true
|
97
82
|
end
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
end
|
113
|
-
|
114
|
-
|
115
|
-
def inherit_errors(object)
|
116
|
-
return unless object.respond_to?(:response_errors) && object.response_errors.respond_to?(:each)
|
117
|
-
object.response_errors.each do |key, messages|
|
118
|
-
if messages.respond_to?(:each)
|
119
|
-
messages.each{|message| errors.add(key, message) }
|
83
|
+
|
84
|
+
def data_row_count_should_match_column_count!
|
85
|
+
# skip validation unless data and column_names present
|
86
|
+
return true unless data? && data.present? && column_names.present?
|
87
|
+
# count the number of expected columns
|
88
|
+
column_count = column_names.count
|
89
|
+
# check each row
|
90
|
+
data.each_with_index do |row, index|
|
91
|
+
# the row is valid if it's count matches the first row's count
|
92
|
+
next if row.count == column_count
|
93
|
+
# the row is invalid if the count is mismatched
|
94
|
+
self.errors.add( :data, "Unexpected number of points in this row:\n#{row.join(',')}\nFound #{row.size-1} but expected #{column_names.count-1} based on precedent from the header row (#{column_names.join(',')})" )
|
95
|
+
# return validation failure
|
96
|
+
return false
|
120
97
|
end
|
98
|
+
true
|
121
99
|
end
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
100
|
+
|
101
|
+
def save_dataset_data
|
102
|
+
return if (!saved? && id.blank?)
|
103
|
+
return if !data? || data.blank?
|
104
|
+
dataset_data.id = id
|
105
|
+
dataset_data.data = benchmark('data.to_csv'){ data.to_csv }
|
106
|
+
benchmark('dataset_data.save'){ dataset_data.save }
|
107
|
+
# update dataset's attributes with dataset_data's attributes
|
108
|
+
attributes.each{|k,v| attributes[k] = dataset_data.attributes[k] if dataset_data.attributes.has_key?(k) }
|
109
|
+
# update dataset errors with dataset_data
|
110
|
+
@metadata[:status] = dataset_data.status unless dataset_data.saved?
|
111
|
+
# inherit_errors(dataset_data) unless dataset_data.saved?
|
112
|
+
end
|
113
|
+
|
114
|
+
|
115
|
+
def inherit_errors(object)
|
116
|
+
return unless object.respond_to?(:response_errors) && object.response_errors.respond_to?(:each)
|
117
|
+
object.response_errors.each do |key, messages|
|
118
|
+
if messages.respond_to?(:each)
|
119
|
+
messages.each{|message| errors.add(key, message) }
|
120
|
+
end
|
121
|
+
end
|
122
|
+
@metadata[:status] = object.status
|
123
|
+
object
|
124
|
+
end
|
125
|
+
|
126
|
+
def enforce_required_formats
|
127
|
+
self.source_code = self.source_code.to_s.upcase
|
128
|
+
self.code = self.code.to_s.upcase
|
129
|
+
self.locations_attributes = locations_attributes.to_json if locations_attributes.respond_to?(:to_json) && !locations_attributes.kind_of?(String)
|
130
|
+
end
|
131
|
+
|
130
132
|
end
|
131
|
-
|
132
133
|
end
|
133
|
-
end
|