quandl_client 2.5.1 → 2.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3018260bc400c7f59310d511e13cb51bfbffe97c
4
- data.tar.gz: 4ab1e74283f845bb3dee3521c18367afc4d9a8ad
3
+ metadata.gz: e5d4902d4e6dbde6094d350de7e43874689b1650
4
+ data.tar.gz: 71e8d0aa7cafac02a3849c69d0aea56055f566bf
5
5
  SHA512:
6
- metadata.gz: 0c1931a9c197ae51d561b1dea40675ba24cca2247262b1c157fcbc78956738d0fd8be26b3c9aee7dd9c8a7c7498336486853debe63e2f0f2a6882b22de6349a5
7
- data.tar.gz: 86c0a7a0f16a237c7cd8d38a1b0ddbfefaaaa675516857740b245374790507392ff457583c57692bc8a946e6a3b307e22bad7eecc6e07452e817910356ff793d
6
+ metadata.gz: 3c1dccdf9bde529231af1bff6d5a268e40dd78e91df934bf11b5dd46e6cca083c1203dcc75bda50e969840ce6182b34ea3ba19a82bee8416fc51bc0c7789a4a8
7
+ data.tar.gz: 4f75457312de981b1046ad1180fd7169d7f22b2d84b53c17494d1f9bf93151d4163febdd3b4341c080707c41585b3c23ba156476a1987a07e705e1b7c2dde062
data/Rakefile CHANGED
@@ -2,6 +2,15 @@ require "bundler"
2
2
  require "rake"
3
3
  require "bundler/gem_tasks"
4
4
  require "rspec/core/rake_task"
5
+ $:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
6
+
7
+ require 'pry'
8
+ require "quandl/client"
9
+
10
+ include Quandl::Client
11
+
12
+ Quandl::Client.use ENV['QUANDL_TEST_URL']
13
+ Quandl::Client.token = ENV['QUANDL_TEST_TOKEN']
5
14
 
6
15
  task :default => :spec
7
16
 
@@ -9,3 +18,7 @@ desc "Run all specs"
9
18
  RSpec::Core::RakeTask.new(:spec) do |task|
10
19
  task.pattern = "spec/**/*_spec.rb"
11
20
  end
21
+
22
+ task :console do |t, args|
23
+ binding.pry
24
+ end
data/UPGRADE.md CHANGED
@@ -1,3 +1,8 @@
1
+ # 2.5.2
2
+
3
+ * WIKI-153 Add scrapers
4
+
5
+
1
6
  # 2.5.1
2
7
 
3
8
  * QUGC-57 move data validation from Quandl::Format into Quandl::Client. Data validations only run on #valid?
@@ -30,6 +30,7 @@ class Quandl::Client::Base
30
30
  Her::API.new.setup url: url_with_version do |c|
31
31
  c.use TokenAuthentication
32
32
  c.use TrackRequestSource
33
+ c.use Faraday::Request::Multipart
33
34
  c.use Faraday::Request::UrlEncoded
34
35
  c.use Quandl::Client::Middleware::ParseJSON
35
36
  c.use Faraday::Adapter::NetHttp
@@ -13,6 +13,8 @@ class Quandl::Client::Dataset < Quandl::Client::Base
13
13
  def find(value)
14
14
  # enforce code formatting
15
15
  if value.is_a?(String)
16
+ # strip extra whitespace
17
+ value = value.strip.rstrip
16
18
  # ensure slashes are forward facing
17
19
  value = value.gsub("\\","/").gsub(".","/")
18
20
  # ensure uppercase
@@ -44,9 +46,9 @@ class Quandl::Client::Dataset < Quandl::Client::Base
44
46
 
45
47
  validates :code, presence: true, format: { with: Quandl::Pattern.code, message: "is invalid. Expected format: #{Quandl::Pattern.code.to_example}" }
46
48
  validates :display_url, allow_blank: true, url: true
49
+ validate :data_row_count_should_match_column_count!
47
50
  validate :data_columns_should_not_exceed_column_names!
48
51
  validate :data_rows_should_have_equal_columns!
49
- validate :data_row_count_should_match_column_count!
50
52
  validate :ambiguous_code_requires_source_code!
51
53
 
52
54
 
@@ -127,7 +129,7 @@ class Quandl::Client::Dataset < Quandl::Client::Base
127
129
 
128
130
  def ambiguous_code_requires_source_code!
129
131
  if code.to_s.numeric? && source_code.blank?
130
- message = %Q{Pure numerical codes like "#{code}" are not allowed unless you prefix them with a source code. Do this:\ncode: <your-username>/#{code}}
132
+ message = %Q{Pure numerical codes like "#{code}" are not allowed unless you include a source code. Do this:\nsource_code: <USERNAME>\ncode: #{code}}
131
133
  self.errors.add( :data, message )
132
134
  return false
133
135
  end
@@ -135,7 +137,7 @@ class Quandl::Client::Dataset < Quandl::Client::Base
135
137
  end
136
138
 
137
139
  def data_columns_should_not_exceed_column_names!
138
- if dataset_data.data? && column_names.present? && data.first.count != column_names.count
140
+ if errors.size == 0 && dataset_data.data? && column_names.present? && data.first.count != column_names.count
139
141
  self.errors.add( :data, "You may not change the number of columns in a dataset. This dataset has #{column_names.count} columns but you tried to send #{data.first.count} columns." )
140
142
  return false
141
143
  end
@@ -152,7 +154,7 @@ class Quandl::Client::Dataset < Quandl::Client::Base
152
154
  # the row is valid if it's count matches the first row's count
153
155
  next if row.count == column_count
154
156
  # the row is invalid if the count is mismatched
155
- self.errors.add( :data, "Unexpected number of points in this row '#{row}'. Expected #{column_count} but found #{row.count} based on #{data[0]}" )
157
+ self.errors.add( :data, "Unexpected number of points in this row:\n#{row.join(',')}\nFound #{row.size-1} but expected #{data[0].size-1} based on precedent from the first row (#{data[0].join(',')})" )
156
158
  # return validation failure
157
159
  return false
158
160
  end
@@ -169,7 +171,7 @@ class Quandl::Client::Dataset < Quandl::Client::Base
169
171
  # the row is valid if it's count matches the first row's count
170
172
  next if row.count == column_count
171
173
  # the row is invalid if the count is mismatched
172
- self.errors.add( :data, "Unexpected number of points in this row '#{row}'. Expected #{column_names.count} but found #{row.count} based on #{column_names}" )
174
+ self.errors.add( :data, "Unexpected number of points in this row:\n#{row.join(',')}\nFound #{row.size-1} but expected #{column_names.count-1} based on precedent from the header row (#{column_names.join(',')})" )
173
175
  # return validation failure
174
176
  return false
175
177
  end
@@ -0,0 +1,11 @@
1
+ module Quandl
2
+ module Client
3
+
4
+ class Location < Quandl::Client::Base
5
+
6
+ attributes :id, :type, :scraper_url
7
+
8
+ end
9
+
10
+ end
11
+ end
@@ -0,0 +1,17 @@
1
+ module Quandl
2
+ module Client
3
+
4
+ class Scraper < Quandl::Client::Base
5
+
6
+ attributes :id, :name, :scraper, :scraper_url, :git_url, :git_reference, :created_at, :updated_at, :type
7
+
8
+ validates :name, presence: true
9
+
10
+ def scraper=(value)
11
+ write_attribute(:scraper, Faraday::UploadIO.new(value, 'text/plain') )
12
+ end
13
+
14
+ end
15
+
16
+ end
17
+ end
@@ -1,6 +1,6 @@
1
1
  module Quandl
2
2
  module Client
3
- VERSION = '2.5.1'
3
+ VERSION = '2.5.2'
4
4
  API_VERSION = 'v2'
5
5
 
6
6
  class << self
data/lib/quandl/client.rb CHANGED
@@ -15,6 +15,8 @@ require 'quandl/client/models/dataset'
15
15
  require 'quandl/client/models/sheet'
16
16
  require 'quandl/client/models/source'
17
17
  require 'quandl/client/models/user'
18
+ require 'quandl/client/models/location'
19
+ require 'quandl/client/models/scraper'
18
20
 
19
21
  module Quandl
20
22
  module Client
@@ -0,0 +1,6 @@
1
+ puts %Q{
2
+ code: SCRAPE_SOME_DATA
3
+ -
4
+ Date,Value
5
+ #{Time.now.strftime("%Y-%m-%d")},10
6
+ }
@@ -13,20 +13,26 @@ describe Dataset do
13
13
  dataset.valid?
14
14
  }
15
15
  its(:valid?){ should be_false }
16
- its('errors.messages'){ should eq({ data: ["Pure numerical codes like \"12345\" are not allowed unless you prefix them with a source code. Do this:\ncode: <your-username>/12345"]}) }
16
+ its('errors.messages'){ should eq({ data: ["Pure numerical codes like \"12345\" are not allowed unless you include a source code. Do this:\nsource_code: <USERNAME>\ncode: 12345"]}) }
17
17
  end
18
18
 
19
19
  context "mismatch row count" do
20
- before(:each){ dataset.data = [[2012, 1,2],[2011, 1,2,3]] }
20
+ before(:each){
21
+ dataset.data = [[2012, 1,2],[2011, 1,2,3]]
22
+ dataset.valid?
23
+ }
21
24
  its(:valid?){ should be_false }
25
+ its('errors.messages'){ should eq({ data: ["Unexpected number of points in this row:\n2011-12-31,1.0,2.0,3.0\nFound 3 but expected 2 based on precedent from the first row (2012-12-31,1.0,2.0)"]}) }
22
26
  end
23
27
 
24
28
  context "mismatch column_names count" do
25
29
  before(:each){
26
30
  dataset.column_names = ['Date','Value']
27
- dataset.data = [[2012, 1,2],[2011, 1,2]]
31
+ dataset.data = [[2012, 18,21],[2011, 1,2]]
32
+ dataset.valid?
28
33
  }
29
34
  its(:valid?){ should be_false }
35
+ its('errors.messages'){ should eq({ data: ["Unexpected number of points in this row:\n2012-12-31,18.0,21.0\nFound 2 but expected 1 based on precedent from the header row (Date,Value)"]}) }
30
36
  end
31
37
 
32
38
  describe "#code" do
@@ -0,0 +1,71 @@
1
+ # encoding: utf-8
2
+ require 'spec_helper'
3
+
4
+ describe Scraper do
5
+
6
+ let(:scraper){ Scraper.new }
7
+ subject{ scraper }
8
+
9
+ its(:valid?){ should be_false }
10
+
11
+ context "name='file'" do
12
+ before(:each){ scraper.name = "file-#{uuid}.rb" }
13
+
14
+ its(:save){ should be_false }
15
+
16
+ context "save" do
17
+ before(:each){ scraper.save }
18
+ its(:error_messages){ should eq( {:response_errors => {"location" => ["You must provide one of: [ scraper_url, git_url ]"]}}) }
19
+ end
20
+
21
+ context "scraper= File" do
22
+ before(:each){
23
+ scraper.scraper = "spec/fixtures/scraper.rb"
24
+ scraper.save
25
+ }
26
+ its(:valid?){ should be_true }
27
+ its(:saved?){ should be_true }
28
+ its(:id){ should be_present }
29
+ its(:scraper_url){ should match /s3/}
30
+ its(:git_url){ should be_blank }
31
+ its(:type){ should eq 'Scraper::Script' }
32
+ end
33
+
34
+ context "scraper_url='https://github.com/tammer/scrapers/blob/master/shibor.rb'" do
35
+ before(:each){
36
+ scraper.scraper_url = "https://github.com/tammer/scrapers/blob/master/shibor.rb"
37
+ scraper.save
38
+ }
39
+ its(:valid?){ should be_true }
40
+ its(:saved?){ should be_true }
41
+ its(:id){ should be_present }
42
+ its(:scraper_url){ should match /s3/}
43
+ its(:git_url){ should be_blank }
44
+ its(:type){ should eq 'Scraper::Script' }
45
+ end
46
+
47
+ context "git_url='git@github.com:tammer/scrapers.git'" do
48
+ before(:each){
49
+ scraper.git_url = "git@github.com:tammer/scrapers.git"
50
+ scraper.save
51
+ }
52
+ its(:error_messages){ should eq({:response_errors=>{"location.reference"=>["can't be blank"]}}) }
53
+ end
54
+
55
+ context "git_url='git@github.com:tammer/scrapers.git' & git_reference='master'" do
56
+ before(:each){
57
+ subject.git_url = "git@github.com:tammer/scrapers-#{uuid}.git"
58
+ subject.git_reference = "master"
59
+ subject.save
60
+ }
61
+ its(:valid?){ should be_true }
62
+ its(:saved?){ should be_true }
63
+ its(:id){ should be_present }
64
+ its(:scraper_url){ should be_blank }
65
+ its(:git_url){ should match /git@github.com:tammer\/scrapers/ }
66
+ its(:type){ should eq 'Scraper::Git' }
67
+ end
68
+
69
+ end
70
+
71
+ end
data/spec/spec_helper.rb CHANGED
@@ -24,4 +24,8 @@ Quandl::Client.use ENV['QUANDL_TEST_URL']
24
24
 
25
25
  RSpec.configure do |config|
26
26
  config.include FactoryGirl::Syntax::Methods
27
+ end
28
+
29
+ def uuid
30
+ SecureRandom.uuid.to_s
27
31
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: quandl_client
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.5.1
4
+ version: 2.5.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Blake Hilscher
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-02-10 00:00:00.000000000 Z
11
+ date: 2014-02-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -210,6 +210,8 @@ files:
210
210
  - lib/quandl/client/middleware/parse_json.rb
211
211
  - lib/quandl/client/models/dataset.rb
212
212
  - lib/quandl/client/models/dataset/data.rb
213
+ - lib/quandl/client/models/location.rb
214
+ - lib/quandl/client/models/scraper.rb
213
215
  - lib/quandl/client/models/sheet.rb
214
216
  - lib/quandl/client/models/source.rb
215
217
  - lib/quandl/client/models/user.rb
@@ -221,6 +223,7 @@ files:
221
223
  - spec/factories/dataset.rb
222
224
  - spec/factories/sheet.rb
223
225
  - spec/factories/source.rb
226
+ - spec/fixtures/scraper.rb
224
227
  - spec/lib/quandl/client/dataset/attributes_spec.rb
225
228
  - spec/lib/quandl/client/dataset/data_spec.rb
226
229
  - spec/lib/quandl/client/dataset/location_spec.rb
@@ -230,6 +233,7 @@ files:
230
233
  - spec/lib/quandl/client/dataset/trim_spec.rb
231
234
  - spec/lib/quandl/client/dataset/validation_spec.rb
232
235
  - spec/lib/quandl/client/dataset_spec.rb
236
+ - spec/lib/quandl/client/scraper_spec.rb
233
237
  - spec/lib/quandl/client/sheet_spec.rb
234
238
  - spec/lib/quandl/client/source_spec.rb
235
239
  - spec/spec_helper.rb
@@ -261,6 +265,7 @@ test_files:
261
265
  - spec/factories/dataset.rb
262
266
  - spec/factories/sheet.rb
263
267
  - spec/factories/source.rb
268
+ - spec/fixtures/scraper.rb
264
269
  - spec/lib/quandl/client/dataset/attributes_spec.rb
265
270
  - spec/lib/quandl/client/dataset/data_spec.rb
266
271
  - spec/lib/quandl/client/dataset/location_spec.rb
@@ -270,6 +275,7 @@ test_files:
270
275
  - spec/lib/quandl/client/dataset/trim_spec.rb
271
276
  - spec/lib/quandl/client/dataset/validation_spec.rb
272
277
  - spec/lib/quandl/client/dataset_spec.rb
278
+ - spec/lib/quandl/client/scraper_spec.rb
273
279
  - spec/lib/quandl/client/sheet_spec.rb
274
280
  - spec/lib/quandl/client/source_spec.rb
275
281
  - spec/spec_helper.rb