quandl_client 2.5.1 → 2.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3018260bc400c7f59310d511e13cb51bfbffe97c
-  data.tar.gz: 4ab1e74283f845bb3dee3521c18367afc4d9a8ad
+  metadata.gz: e5d4902d4e6dbde6094d350de7e43874689b1650
+  data.tar.gz: 71e8d0aa7cafac02a3849c69d0aea56055f566bf
 SHA512:
-  metadata.gz: 0c1931a9c197ae51d561b1dea40675ba24cca2247262b1c157fcbc78956738d0fd8be26b3c9aee7dd9c8a7c7498336486853debe63e2f0f2a6882b22de6349a5
-  data.tar.gz: 86c0a7a0f16a237c7cd8d38a1b0ddbfefaaaa675516857740b245374790507392ff457583c57692bc8a946e6a3b307e22bad7eecc6e07452e817910356ff793d
+  metadata.gz: 3c1dccdf9bde529231af1bff6d5a268e40dd78e91df934bf11b5dd46e6cca083c1203dcc75bda50e969840ce6182b34ea3ba19a82bee8416fc51bc0c7789a4a8
+  data.tar.gz: 4f75457312de981b1046ad1180fd7169d7f22b2d84b53c17494d1f9bf93151d4163febdd3b4341c080707c41585b3c23ba156476a1987a07e705e1b7c2dde062

data/Rakefile

@@ -2,6 +2,15 @@ require "bundler"
 require "rake"
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+
+require 'pry'
+require "quandl/client"
+
+include Quandl::Client
+
+Quandl::Client.use ENV['QUANDL_TEST_URL']
+Quandl::Client.token = ENV['QUANDL_TEST_TOKEN']
 
 task :default => :spec
 
@@ -9,3 +18,7 @@ desc "Run all specs"
 RSpec::Core::RakeTask.new(:spec) do |task|
   task.pattern = "spec/**/*_spec.rb"
 end
+
+task :console do |t, args|
+  binding.pry
+end

data/UPGRADE.md

@@ -1,3 +1,8 @@
+# 2.5.2
+
+* WIKI-153 Add scrapers
+
+
 # 2.5.1
 
 * QUGC-57 move data validation from Quandl::Format into Quandl::Client. Data validations only run on #valid?

data/lib/quandl/client/base.rb

@@ -30,6 +30,7 @@ class Quandl::Client::Base
       Her::API.new.setup url: url_with_version do |c|
         c.use TokenAuthentication
         c.use TrackRequestSource
+        c.use Faraday::Request::Multipart
         c.use Faraday::Request::UrlEncoded
         c.use Quandl::Client::Middleware::ParseJSON
         c.use Faraday::Adapter::NetHttp

data/lib/quandl/client/models/dataset.rb

@@ -13,6 +13,8 @@ class Quandl::Client::Dataset < Quandl::Client::Base
     def find(value)
       # enforce code formatting
       if value.is_a?(String)
+        # strip extra whitespace
+        value = value.strip.rstrip
         # ensure slashes are forward facing
         value = value.gsub("\\","/").gsub(".","/")
         # ensure uppercase
@@ -44,9 +46,9 @@ class Quandl::Client::Dataset < Quandl::Client::Base
   
   validates :code, presence: true, format: { with: Quandl::Pattern.code, message: "is invalid. Expected format: #{Quandl::Pattern.code.to_example}" }
   validates :display_url, allow_blank: true, url: true
+  validate :data_row_count_should_match_column_count!
   validate :data_columns_should_not_exceed_column_names!
   validate :data_rows_should_have_equal_columns!
-  validate :data_row_count_should_match_column_count!
   validate :ambiguous_code_requires_source_code!
   
   
@@ -127,7 +129,7 @@ class Quandl::Client::Dataset < Quandl::Client::Base
   
   def ambiguous_code_requires_source_code!
     if code.to_s.numeric? && source_code.blank?
-      message = %Q{Pure numerical codes like "#{code}" are not allowed unless you prefix them with a source code. Do this:\ncode: <your-username>/#{code}}
+      message = %Q{Pure numerical codes like "#{code}" are not allowed unless you include a source code. Do this:\nsource_code: <USERNAME>\ncode: #{code}}
       self.errors.add( :data, message )
       return false
     end
@@ -135,7 +137,7 @@ class Quandl::Client::Dataset < Quandl::Client::Base
   end
   
   def data_columns_should_not_exceed_column_names!
-    if dataset_data.data? && column_names.present? && data.first.count != column_names.count
+    if errors.size == 0 && dataset_data.data? && column_names.present? && data.first.count != column_names.count
       self.errors.add( :data, "You may not change the number of columns in a dataset. This dataset has #{column_names.count} columns but you tried to send #{data.first.count} columns." )
       return false
     end
@@ -152,7 +154,7 @@ class Quandl::Client::Dataset < Quandl::Client::Base
       # the row is valid if it's count matches the first row's count
       next if row.count == column_count
       # the row is invalid if the count is mismatched
-      self.errors.add( :data, "Unexpected number of points in this row '#{row}'. Expected #{column_count} but found #{row.count} based on #{data[0]}" )
+      self.errors.add( :data, "Unexpected number of points in this row:\n#{row.join(',')}\nFound #{row.size-1} but expected #{data[0].size-1} based on precedent from the first row (#{data[0].join(',')})" )
       # return validation failure
       return false
     end
@@ -169,7 +171,7 @@ class Quandl::Client::Dataset < Quandl::Client::Base
       # the row is valid if it's count matches the first row's count
       next if row.count == column_count
       # the row is invalid if the count is mismatched
-      self.errors.add( :data, "Unexpected number of points in this row '#{row}'. Expected #{column_names.count} but found #{row.count} based on #{column_names}" )
+      self.errors.add( :data, "Unexpected number of points in this row:\n#{row.join(',')}\nFound #{row.size-1} but expected #{column_names.count-1} based on precedent from the header row (#{column_names.join(',')})" )
       # return validation failure
       return false
     end

data/lib/quandl/client/models/location.rb

@@ -0,0 +1,11 @@
+module Quandl
+module Client
+
+class Location < Quandl::Client::Base
+  
+  attributes :id, :type, :scraper_url
+  
+end
+
+end
+end

data/lib/quandl/client/models/scraper.rb

@@ -0,0 +1,17 @@
+module Quandl
+module Client
+
+class Scraper < Quandl::Client::Base
+  
+  attributes :id, :name, :scraper, :scraper_url, :git_url, :git_reference, :created_at, :updated_at, :type
+  
+  validates :name, presence: true
+  
+  def scraper=(value)
+    write_attribute(:scraper, Faraday::UploadIO.new(value, 'text/plain') )
+  end
+  
+end
+
+end
+end

data/lib/quandl/client/version.rb

@@ -1,6 +1,6 @@
 module Quandl
 module Client
-  VERSION = '2.5.1'
+  VERSION = '2.5.2'
   API_VERSION = 'v2'
   
   class << self

data/lib/quandl/client.rb

@@ -15,6 +15,8 @@ require 'quandl/client/models/dataset'
 require 'quandl/client/models/sheet'
 require 'quandl/client/models/source'
 require 'quandl/client/models/user'
+require 'quandl/client/models/location'
+require 'quandl/client/models/scraper'
 
 module Quandl
   module Client

data/spec/fixtures/scraper.rb

@@ -0,0 +1,6 @@
+puts %Q{
+code: SCRAPE_SOME_DATA
+-
+Date,Value
+#{Time.now.strftime("%Y-%m-%d")},10
+}

data/spec/lib/quandl/client/dataset/validation_spec.rb

@@ -13,20 +13,26 @@ describe Dataset do
       dataset.valid?
     }
     its(:valid?){ should be_false }
-    its('errors.messages'){ should eq({ data: ["Pure numerical codes like \"12345\" are not allowed unless you prefix them with a source code. Do this:\ncode: <your-username>/12345"]}) }
+    its('errors.messages'){ should eq({ data: ["Pure numerical codes like \"12345\" are not allowed unless you include a source code. Do this:\nsource_code: <USERNAME>\ncode: 12345"]}) }
   end
   
   context "mismatch row count" do
-    before(:each){ dataset.data = [[2012, 1,2],[2011, 1,2,3]] }
+    before(:each){
+      dataset.data = [[2012, 1,2],[2011, 1,2,3]]
+      dataset.valid?
+    }
     its(:valid?){ should be_false }
+    its('errors.messages'){ should eq({ data: ["Unexpected number of points in this row:\n2011-12-31,1.0,2.0,3.0\nFound 3 but expected 2 based on precedent from the first row (2012-12-31,1.0,2.0)"]}) }
   end
   
   context "mismatch column_names count" do
     before(:each){
       dataset.column_names = ['Date','Value']
-      dataset.data = [[2012, 1,2],[2011, 1,2]]
+      dataset.data = [[2012, 18,21],[2011, 1,2]]
+      dataset.valid?
     }
     its(:valid?){ should be_false }
+    its('errors.messages'){ should eq({ data: ["Unexpected number of points in this row:\n2012-12-31,18.0,21.0\nFound 2 but expected 1 based on precedent from the header row (Date,Value)"]}) }
   end
   
   describe "#code" do

data/spec/lib/quandl/client/scraper_spec.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Scraper do
+  
+  let(:scraper){ Scraper.new }
+  subject{ scraper }
+  
+  its(:valid?){ should be_false }
+  
+  context "name='file'" do
+    before(:each){ scraper.name = "file-#{uuid}.rb" }
+    
+    its(:save){ should be_false }
+    
+    context "save" do
+      before(:each){ scraper.save }
+      its(:error_messages){ should eq( {:response_errors => {"location" => ["You must provide one of: [ scraper_url, git_url ]"]}}) }
+    end
+    
+    context "scraper= File" do
+      before(:each){ 
+        scraper.scraper = "spec/fixtures/scraper.rb"
+        scraper.save
+      }
+      its(:valid?){ should be_true }
+      its(:saved?){ should be_true }
+      its(:id){ should be_present }
+      its(:scraper_url){ should match /s3/}
+      its(:git_url){ should be_blank }
+      its(:type){ should eq 'Scraper::Script' }
+    end
+    
+    context "scraper_url='https://github.com/tammer/scrapers/blob/master/shibor.rb'" do
+      before(:each){ 
+        scraper.scraper_url = "https://github.com/tammer/scrapers/blob/master/shibor.rb" 
+        scraper.save
+      }
+      its(:valid?){ should be_true }
+      its(:saved?){ should be_true }
+      its(:id){ should be_present }
+      its(:scraper_url){ should match /s3/}
+      its(:git_url){ should be_blank }
+      its(:type){ should eq 'Scraper::Script' }
+    end
+    
+    context "git_url='git@github.com:tammer/scrapers.git'" do
+      before(:each){ 
+        scraper.git_url = "git@github.com:tammer/scrapers.git" 
+        scraper.save
+      }
+      its(:error_messages){ should eq({:response_errors=>{"location.reference"=>["can't be blank"]}}) }
+    end
+    
+    context "git_url='git@github.com:tammer/scrapers.git' & git_reference='master'" do
+      before(:each){
+        subject.git_url = "git@github.com:tammer/scrapers-#{uuid}.git" 
+        subject.git_reference = "master" 
+        subject.save
+      }
+      its(:valid?){ should be_true }
+      its(:saved?){ should be_true }
+      its(:id){ should be_present }
+      its(:scraper_url){ should be_blank }
+      its(:git_url){ should match /git@github.com:tammer\/scrapers/ }
+      its(:type){ should eq 'Scraper::Git' }
+    end
+    
+  end
+  
+end

data/spec/spec_helper.rb

@@ -24,4 +24,8 @@ Quandl::Client.use ENV['QUANDL_TEST_URL']
 
 RSpec.configure do |config|
   config.include FactoryGirl::Syntax::Methods
+end
+
+def uuid
+  SecureRandom.uuid.to_s
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quandl_client
 version: !ruby/object:Gem::Version
-  version: 2.5.1
+  version: 2.5.2
 platform: ruby
 authors:
 - Blake Hilscher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-10 00:00:00.000000000 Z
+date: 2014-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -210,6 +210,8 @@ files:
 - lib/quandl/client/middleware/parse_json.rb
 - lib/quandl/client/models/dataset.rb
 - lib/quandl/client/models/dataset/data.rb
+- lib/quandl/client/models/location.rb
+- lib/quandl/client/models/scraper.rb
 - lib/quandl/client/models/sheet.rb
 - lib/quandl/client/models/source.rb
 - lib/quandl/client/models/user.rb
@@ -221,6 +223,7 @@ files:
 - spec/factories/dataset.rb
 - spec/factories/sheet.rb
 - spec/factories/source.rb
+- spec/fixtures/scraper.rb
 - spec/lib/quandl/client/dataset/attributes_spec.rb
 - spec/lib/quandl/client/dataset/data_spec.rb
 - spec/lib/quandl/client/dataset/location_spec.rb
@@ -230,6 +233,7 @@ files:
 - spec/lib/quandl/client/dataset/trim_spec.rb
 - spec/lib/quandl/client/dataset/validation_spec.rb
 - spec/lib/quandl/client/dataset_spec.rb
+- spec/lib/quandl/client/scraper_spec.rb
 - spec/lib/quandl/client/sheet_spec.rb
 - spec/lib/quandl/client/source_spec.rb
 - spec/spec_helper.rb
@@ -261,6 +265,7 @@ test_files:
 - spec/factories/dataset.rb
 - spec/factories/sheet.rb
 - spec/factories/source.rb
+- spec/fixtures/scraper.rb
 - spec/lib/quandl/client/dataset/attributes_spec.rb
 - spec/lib/quandl/client/dataset/data_spec.rb
 - spec/lib/quandl/client/dataset/location_spec.rb
@@ -270,6 +275,7 @@ test_files:
 - spec/lib/quandl/client/dataset/trim_spec.rb
 - spec/lib/quandl/client/dataset/validation_spec.rb
 - spec/lib/quandl/client/dataset_spec.rb
+- spec/lib/quandl/client/scraper_spec.rb
 - spec/lib/quandl/client/sheet_spec.rb
 - spec/lib/quandl/client/source_spec.rb
 - spec/spec_helper.rb