quandl_format 0.2.8 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c2355e2207291cd09c76a495b2565c28fefdbd0
-  data.tar.gz: 5699891dd773b8d58e4824d4a7411b39db054e92
+  metadata.gz: d3736904b3e0d1c5ee9b90ebfaca235563775f56
+  data.tar.gz: f12745b06559794e379a999280cb4e6c28ea01ae
 SHA512:
-  metadata.gz: 6426c47bc68a9891942a529ba2220a3360de14031e68f7b8d117a5faaf01cad4ba8e30c406b6c101778c09a6a33dddb7f04037ef26b5ceff4a4807f4b6e35606
-  data.tar.gz: a0db81d386ea365f37d2f64007c04f37ec6e4cb7f0fdb577905ae6824eaeded19545c675bf7ac20f52d2574d989e0613a7a7bcd8548598443a2e76ed0dc584e8
+  metadata.gz: 3537467133b0cb0f5b89732974bc034b2c5198970d2b215789a53d2a714ce63a8d09665b0d0afcd6a8b424404e07e7cdb199f34f3fe187ab8ec621ad4facc1d0
+  data.tar.gz: a4ad1d3c31abd11d3ee0a493f1e3126f28db56d6b7d124aec9825b48465b046ce3edabbd41aa596f71e88c89a51d8575f39a3695c49fdd80c423c3a17fbfbfc9

data/Rakefile

@@ -2,6 +2,8 @@ require "bundler"
 require "rake"
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require 'quandl/format'
+require 'pry'
 
 task :default => :spec
 
@@ -9,3 +11,15 @@ desc "Run all specs"
 RSpec::Core::RakeTask.new(:spec) do |task|
   task.pattern = "spec/**/*_spec.rb"
 end
+
+task :console do |t,args|
+  binding.pry
+end
+
+require 'quandl/utility/rake_tasks'
+Quandl::Utility::Tasks.configure do |c|
+  c.name              = 'quandl_format'
+  c.version_path      = 'VERSION'
+  c.changelog_path    = 'UPGRADE.md'
+  c.changelog_matching  = ['^QUGC','^WIKI']
+end

data/UPGRADE.md

@@ -1,3 +1,10 @@
+## 0.3.0 
+
+* QUGC-104 bump gems to get onboard with new validations and error handling
+* QUGC-104 Working towards a solution where all dataset validation happens in one place.
+
+
+
 ## 0.2.8
 
 * QUGC-57 validation at an inappropriate time; move validations to Quandl::Client

data/VERSION

@@ -0,0 +1 @@
+0.3.0

data/lib/quandl/format/dataset/attributes.rb

@@ -60,11 +60,11 @@ module Attributes
 
   def data=(rows)
     rows = rows.to_table if rows.respond_to?(:to_table)
-    @data = Quandl::Data.new(rows).to_date
+    @data = Quandl::Data.new(rows)
     self.column_names = @data.headers if @data.headers.present?
     @data
   end
-  
+
   def column_names
     @column_names ||= []
   end
@@ -84,9 +84,9 @@ module Attributes
   def attributes
     self.class.attribute_names.inject({}){|m,k| m[k] = self.send(k) unless self.send(k).nil?; m }
   end
-  
+
   private
-  
+
   def raise_unknown_attribute_error!(key)
     m = "Unknown Field '#{key}' valid fields are: #{self.class.meta_attribute_names.join(', ')}"
     raise Quandl::Error::UnknownAttribute, m

data/lib/quandl/format/dataset/client.rb

@@ -5,6 +5,13 @@ class Dataset
 module Client
   extend ActiveSupport::Concern
   
+  included do
+    include ActiveModel::Validations
+    
+    validate :client_should_be_valid!
+    
+  end
+  
   def human_errors
     m = "#{client.human_status} \t #{client.full_url}"
     return m if errors.blank?
@@ -28,15 +35,6 @@ module Client
     client.save if valid?
   end
   
-  def errors
-    client.error_messages
-  end
-  
-  def valid?
-    assign_client_attributes
-    client.valid_with_server?
-  end
-  
   def client
     @client ||= find_or_build_client
   end
@@ -48,13 +46,18 @@ module Client
   
   protected
   
-  def assign_client_attributes
-    client.assign_attributes(attributes)
+  def client_should_be_valid!
+    if !client.valid_with_server?
+      client.errors.each{|err, value| self.errors.add( err, value ) }
+      return false 
+    end
+    true
   end
   
   def find_or_build_client
-    @client = Quandl::Client::Dataset.find(full_code)
+    @client ||= Quandl::Client::Dataset.find(full_code)
     @client = Quandl::Client::Dataset.new unless @client.try(:exists?)
+    @client.assign_attributes(attributes)
     @client
   end
   

data/lib/quandl/format/dataset/load.rb

@@ -89,15 +89,22 @@ class Quandl::Format::Dataset::Load
     end
     
     def process_node(node, &block)
-      node = parse_node(node)
-      # fail on errored node
-      return false if node == false
-      # convert node to dataset
-      dataset = convert_node_to_dataset(node)
-      # do whatever we need to do with the node
-      block.call( dataset ) unless dataset.nil?
-      # success
-      true
+      begin
+        node = parse_node(node)
+        # fail on errored node
+        return false if node == false
+        # convert node to dataset
+        dataset = convert_node_to_dataset(node)
+        # do whatever we need to do with the node
+        block.call( dataset, nil ) unless dataset.nil?
+        # success
+        true
+        
+      rescue Exception => err
+        block.call( nil, err )
+        false
+        
+      end
     end
     
     def parse_node(node)
@@ -121,8 +128,8 @@ class Quandl::Format::Dataset::Load
       end
       attrs
     rescue Exception => err
-      log_yaml_parse_error(node, err)
-      nil
+      m = generate_yaml_parse_error(node, err)
+      raise err, m
     end
     
     def convert_node_to_dataset(node)
@@ -130,32 +137,40 @@ class Quandl::Format::Dataset::Load
       dataset.data = node[:data]
       dataset
     rescue Exception => err
-      log_dataset_error(node, err)
-      nil
+      m = generate_dataset_error(node, err)
+      raise err, m
     end
     
-    def log_yaml_parse_error(node, err)
+    def generate_yaml_parse_error(node, err)
       message = ""
       if err.message == 'Unparsable input'
         message = "Input data is unparsable.  Are you missing a colon (:) or a space after a colon?\n"
-      elsif err.is_a?(Psych::SyntaxError)
+      elsif err.is_a?(Psych::SyntaxError) && err.respond_to?(:problem)
         if err.problem =~ /mapping values are not allowed in this context/
-          message = "Syntax error before line #{1+node[:offset] + err.line}.  Are you missing a colon (:) or a space after a colon?\n"
+          message = "Syntax error *before* line #{1+node[:offset] + err.line}.\n"
+          if node[:attributes] =~ /:.+:/ # he probably has a colon in a field.
+            message += "You might have an illegal colon (:) in one of your fields.  If so, use quotes.\n"
+          elsif node[:attributes] =~ /^([^:]+)$/ # he forgot the colon completely
+            message += "Did you forget a colon on this line:\n"
+            message += "#{$1}\n"
+          end
         else
           message += "Error parsing metadata. #{err.problem.capitalize} on line #{node[:offset] + err.line}\n"
           if err.problem =~ /expected ':'/
             message += "Did you forget to delimit the meta data section from the data section with a one or more dashes ('#{SYNTAX[:data]}')?\n"
           end
         end
+      elsif err.is_a?(Psych::SyntaxError)
+        message = err.to_s + "\n" + node[:attributes]
       else
         message += "Attribute parse error at line #{ node[:line] + err.line } column #{err.column}. #{err.problem} (#{err.class})\n" if node.has_key?(:line) && err.respond_to?(:line)
-        message += "Encountered error while parsing: \n  " + node[:attributes].split("\n")[err.line - 1].to_s + "\n" if err.respond_to?(:line)
+        message += "Encountered error while parsing: \n  " + node[:attributes].split("\n")[err.line - 1].to_s if err.respond_to?(:line)
       end
-      message += "--"
-      Quandl::Logger.error(message)
+      message += "\n"
+      message
     end
     
-    def log_dataset_error( node, err )
+    def generate_dataset_error( node, err )
       message = ''
       message += node[:attributes][:source_code] + '/' if node[:attributes][:source_code].present?
       message += node[:attributes][:code] + ' '
@@ -166,9 +181,9 @@ class Quandl::Format::Dataset::Load
         message += "error around line #{node[:line]}\n"
       end
       # include original error
-      message += "#{$!} (#{err.class})\n"
-      message += "--"
-      Quandl::Logger.error(message)
+      message += "#{$!} (#{err.class})"
+      message += "\n"
+      message
     end
     
   end

data/lib/quandl/format/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Format
-  VERSION = "0.2.8"
+  VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), '../../../VERSION'))).strip.rstrip
 end
 end

data/quandl_format.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
   
-  s.add_runtime_dependency "quandl_client", "~> 2.4"
+  s.add_runtime_dependency "quandl_client", "~> 2.6"
   
   s.add_development_dependency "rake", "~> 10.0"
   s.add_development_dependency "rspec", "~> 2.13"
@@ -26,4 +26,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency "simplecov"
   s.add_development_dependency "guard"
   s.add_development_dependency "guard-rspec"
+  s.add_development_dependency "quandl_utility"
 end

data/spec/fixtures/data/illegal_dash.qdf

@@ -0,0 +1,5 @@
+source_code: SERGEI
+code: S1
+–
+Date,Value
+dede,12

data/spec/fixtures/data/invalid_date.qdf

@@ -0,0 +1,12 @@
+code: "BLAKE_8"
+name: "A new title"
+description: "The description Date, Open, High"
+private: false
+-
+Date,Open,High,Low
+2013,10,20,30
+2012,10,40,30
+2011,10,60,30
+2010,10,80,30
+ASDF,10,100,30
+2008,10,120,30

data/spec/lib/quandl/format/dataset/load/errors_spec.rb

@@ -2,21 +2,50 @@
 require 'spec_helper'
 
 describe Quandl::Format::Dataset do
+    
+  let(:file_path){ 'spec/fixtures/data/' }
+  let(:file){ File.open( File.join(file_path, self.class.superclass.description + '.qdf')) }
+  let(:output){
+    output = []
+    Quandl::Format::Dataset.each_line( file ){|r,e| output << OpenStruct.new( record: r, error: e ) }
+    output
+  }
+  subject{ output.first }
+  
+  context "unknown_attribute" do
+    its(:record){ should be_nil }
+    its("error.to_s"){ should match /this_attribute_does_not_exist/ }
+  end
 
-  def self.it_should_expect_error(file, error)
-    it "#{file}.qdf should error with #{error}" do
-      Quandl::Logger.should_receive(:error).at_least(:once).with(error)
-      Quandl::Format::Dataset.load( fixtures_data[file] )
-    end
+  context "invalid_yaml" do
+    its(:record){ should be_nil }
+    its("error.to_s"){ should match /Could not find expected ':'/ }
   end
-  
-  it_should_expect_error 'invalid_data',       /Date/
-  it_should_expect_error 'unknown_attribute',  /this_attribute_does_not_exist/
-  it_should_expect_error 'invalid_yaml',       /Could not find expected ':'/
-  it_should_expect_error 'missing_dashes',     /Could not find expected ':' on line 22/
-  it_should_expect_error 'missing_dashes',     /Did you forget to delimit the meta data section/
-  it_should_expect_error 'missing_colon',     /Are you missing a colon/
-  it_should_expect_error 'missing_colon2',     /Could not find expected ':' on line 3/
-  it_should_expect_error 'missing_space',     /Are you missing a colon/
-  
+
+  context "illegal_dash" do
+    its(:record){ should be_nil }
+    its("error.to_s"){ should match /Could not find expected ':'/ }
+  end
+
+  context "missing_dashes" do
+    subject{ output[2] }
+    its(:record){ should be_nil }
+    its("error.to_s"){ should match /Could not find expected ':' on line 22/ }
+  end
+
+  context "missing_colon" do
+    its(:record){ should be_nil }
+    its("error.to_s"){ should match /Did you forget a colon on this line/ }
+  end
+
+  context "missing_colon2" do
+    its(:record){ should be_nil }
+    its("error.to_s"){ should match /Could not find expected ':' on line 3/ }
+  end
+
+  context "missing_space" do
+    its(:record){ should be_nil }
+    its("error.to_s"){ should match /Are you missing a colon/ }
+  end
+ 
 end

data/spec/lib/quandl/format/dataset/load/validation_spec.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Format::Dataset do
+  
+  let(:file){ self.class.superclass.description }
+  subject{ Quandl::Format::Dataset.load( fixtures_data[file] ).first }
+  
+  context "invalid_data" do
+    before(:each){ subject.valid? }
+    its(:valid?){ should be_false }
+    its('errors.messages'){ should eq({ data: ["Invalid date segments. Expected yyyy-mm-dd received 'Date'"] }) }
+  end
+  
+  context "invalid_date" do
+    before(:each){ subject.valid? }
+    its(:valid?){ should be_false }
+    
+    its('errors.messages'){ should eq({ data: ["Invalid date 'ASDF'"] }) }
+    its('client.valid?'){ should be_false }
+    its('client.errors.messages'){ should eq({ data: ["Invalid date 'ASDF'"] }) }
+    its('client.data.valid?'){ should be_false }
+    its('client.data.errors.messages'){ should eq({ data: ["Invalid date 'ASDF'"] }) }
+  end
+  
+end

data/spec/lib/quandl/format/dataset/load_spec.rb

@@ -8,7 +8,7 @@ describe Quandl::Format::Dataset::Load do
   describe "junk" do
     let(:collection){ Quandl::Format::Dataset::Load.string('asdf') }
     subject{ collection }
-    its(:count){ should eq 0 }
+    its(:count){ should eq 1 }
   end
   
   describe ".file" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quandl_format
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.0
 platform: ruby
 authors:
 - Blake Hilscher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-10 00:00:00.000000000 Z
+date: 2014-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: quandl_client
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: '2.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +122,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: quandl_utility
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Data will be loaded and dumped.
 email:
 - blake@hilscher.ca
@@ -136,6 +150,7 @@ files:
 - README.md
 - Rakefile
 - UPGRADE.md
+- VERSION
 - examples/load.rb
 - lib/quandl/client/dataset/to_qdf.rb
 - lib/quandl/error/column_count_mismatch.rb
@@ -152,7 +167,9 @@ files:
 - spec/config/logger.rb
 - spec/fixtures/data/annual.qdf
 - spec/fixtures/data/illegal_colon.qdf
+- spec/fixtures/data/illegal_dash.qdf
 - spec/fixtures/data/invalid_data.qdf
+- spec/fixtures/data/invalid_date.qdf
 - spec/fixtures/data/invalid_yaml.qdf
 - spec/fixtures/data/metadata_only.qdf
 - spec/fixtures/data/mismatched_columns.qdf
@@ -170,6 +187,7 @@ files:
 - spec/lib/quandl/format/dataset/client_spec.rb
 - spec/lib/quandl/format/dataset/load/errors_spec.rb
 - spec/lib/quandl/format/dataset/load/valid_spec.rb
+- spec/lib/quandl/format/dataset/load/validation_spec.rb
 - spec/lib/quandl/format/dataset/load_spec.rb
 - spec/lib/quandl/format/dataset_spec.rb
 - spec/spec_helper.rb
@@ -202,7 +220,9 @@ test_files:
 - spec/config/logger.rb
 - spec/fixtures/data/annual.qdf
 - spec/fixtures/data/illegal_colon.qdf
+- spec/fixtures/data/illegal_dash.qdf
 - spec/fixtures/data/invalid_data.qdf
+- spec/fixtures/data/invalid_date.qdf
 - spec/fixtures/data/invalid_yaml.qdf
 - spec/fixtures/data/metadata_only.qdf
 - spec/fixtures/data/mismatched_columns.qdf
@@ -220,6 +240,7 @@ test_files:
 - spec/lib/quandl/format/dataset/client_spec.rb
 - spec/lib/quandl/format/dataset/load/errors_spec.rb
 - spec/lib/quandl/format/dataset/load/valid_spec.rb
+- spec/lib/quandl/format/dataset/load/validation_spec.rb
 - spec/lib/quandl/format/dataset/load_spec.rb
 - spec/lib/quandl/format/dataset_spec.rb
 - spec/spec_helper.rb