quandl_format 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 30e3f912e78fe6901d62321ed87be42b9986eba6
-  data.tar.gz: 19b1cf7491961530d8a467fae867fd0d0d85ec5c
+  metadata.gz: 3d570d30030c6efc98ca36991c9f7bd696ef1b06
+  data.tar.gz: a3055d3a66e3189abdee26cc3cd30454c78919aa
 SHA512:
-  metadata.gz: a340198d9d890029b5d4bbb939e8f13e9abcae97fda791b88f35727e28b2a73face0c22e10efb0be6f49de5ca9bc14fcfc15ee4228f9a3ed85f56c79fe5f3077
-  data.tar.gz: 63015d53d810bc61a893a50902e2eb6a3ba0ea20ff5773fbd2b9f086570cfc1b4e75fa1909119f61758c1082d542c77c5797aaaa57d5e813b27e14de43003902
+  metadata.gz: 65deb544f8ff59f3cb623cfa0672319c48b39a5243e20837d196ce8057eb3523a7908d4bbd801a14b796de6d47ba88a7c09e82c97a507357774018c7b6a92d1a
+  data.tar.gz: cd66e027d2cbfa58b0e6e23b1b1369891101053715fe046de229c4202f8ac5a39d6737629a4954d97eba5bf1d23f217345cc4a071b30cfeafb4ed896416dcab9

data/UPGRADE.md

@@ -1,3 +1,9 @@
+## 0.1.6
+
+* refactor errors to include specific line numbers
+* revise errors to be more human friendly
+
+
 ## 0.1.5
 
 * Replace Quandl::Babelfish::Data with Quandl::Data, now that QD does intelligent cleaning.

data/lib/quandl/error/column_count_mismatch.rb

@@ -1,7 +1,7 @@
 module Quandl
 module Error
   
-  class ColumnCountMismatch < StandardError; end
+  class ColumnCountMismatch < Quandl::Error::Standard; end
   
 end
 end

data/lib/quandl/format/dataset/attributes.rb

@@ -103,17 +103,17 @@ module Attributes
   private
   
   def raise_row_column_mismatch!(row, index)
-    m = "ColumnCountMismatch #{full_code} data[0] had #{data[0].count} columns, but data[#{index}] had #{row.count} #{row}"
-    raise Quandl::Error::ColumnCountMismatch, m    
+    m = "Unexpected number of points in this row '#{row}'. Expected #{data[0].count} but found #{row.count} based on #{data[0]}"
+    raise Quandl::Error::ColumnCountMismatch.new( row: row, line: index, context: 'data_rows_should_have_equal_columns!' ), m
   end
   
   def raise_column_count_mismatch!(row, index)
-    m = "ColumnCountMismatch #{full_code} column_names had #{column_names.count} columns, but data[#{index}] had #{row.count} #{row}"
-    raise Quandl::Error::ColumnCountMismatch, m
+    m = "Unexpected number of points in this row '#{row}'. Expected #{column_names.count} but found #{row.count} based on #{column_names}"
+    raise Quandl::Error::ColumnCountMismatch.new( row: row, line: index+1, context: 'data_row_count_should_match_column_count!' ), m
   end
   
   def raise_unknown_attribute_error!(key)
-    m = "UnknownAttribute #{key} recognized attributes are: #{self.class.attribute_names}"
+    m = "Unknown Field '#{key}' valid fields are: #{self.class.attribute_names.join(', ')}"
     raise Quandl::Error::UnknownAttribute, m
   end
 

data/lib/quandl/format/dataset/load.rb

@@ -38,8 +38,9 @@ class Quandl::Format::Dataset::Load
           section_type = :attributes
           
           # have we reached the end of the attributes?
-        elsif line[0] == '-'
+        elsif line == '-'
           # update the section to data
+          nodes[-1][:data_line] = line_index + 1
           section_type = :data
           # skip to the next line
           next
@@ -77,31 +78,45 @@ class Quandl::Format::Dataset::Load
     
     def parse_yaml_attributes(node)
       YAML.load( node[:attributes] ).symbolize_keys!
-    rescue => e
-      message = "Error: Dataset starting at line #{node[:line]}\n"
-      message += "#{$!}\n"
-      message += "--"
-      Quandl::Logger.error(message)
+    rescue => err
+      log_yaml_parse_error(node, err)
       nil
     end
     
     def node_to_dataset(node)
       Quandl::Format::Dataset.new( node[:attributes] )
-    rescue => e
+    rescue => err
+      log_dataset_error(node, err)
+    end
+    
+    def attribute_format
+      /^([a-z0-9_]+): (.+)/
+    end
+    
+    def log_yaml_parse_error(node, err)
+      message = "Attribute parse error at line #{ node[:line] + err.line } column #{err.column}. #{err.problem} (#{err.class})\n"
+      message += "Did you forget to delimit the meta data section from the data section with a one or more dashes ('-')?\n" unless node[:attributes] =~ /^-/
+      message += "--"
+      Quandl::Logger.error(message)
+    end
+    
+    def log_dataset_error( node, err )
       message = ''
       message += node[:attributes][:source_code] + '/' if node[:attributes][:source_code].present?
       message += node[:attributes][:code] + ' '
-      message += "error around line #{node[:line]} \n"
-      message += "#{$!}\n"
+      # include specific line if available
+      if err.respond_to?(:line)
+        message += "error at line #{node[:data_line].to_i + err.line.to_i}\n"
+      else
+        message += "error around line #{node[:line]}\n"
+      end
+      # include original error
+      message += "#{$!} (#{err.class})\n"
       message += "--"
       Quandl::Logger.error(message)
       nil
     end
     
-    def attribute_format
-      /^([a-z0-9_]+): (.+)/
-    end
-  
   end
 
 end

data/lib/quandl/format/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Format
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end
 end

data/spec/fixtures/data/missing_dashes.qdf

@@ -0,0 +1,21 @@
+code: "BLAKE_TEST_1"
+name: "A new title"
+description: "The description Date, Open, High"
+private: false
+Date,Open,High,Low,Last,Close,Total Trade Quantity,Turnover (Lacs)
+2013-11-22,1252.0,454.95,448.2,450.0,450.0,1354405.0,6099.41
+2013-11-21,452.25,457.75,449.1,451.2,451.0,218881.0,992.94
+
+code: "BLAKE_TEST_2"
+name: "A new title"
+description: "The description Date, Open, High"
+Date,Open,High,Low,Last,Close,Total Trade Quantity,Turnover (Lacs)
+2013-11-22,1252.0,454.95,448.2,450.0,450.0,1354405.0,6099.41
+2013-11-21,452.25,457.75,449.1,451.2,451.0,218881.0,992.94
+
+code: "BLAKE_TEST_3"
+name: "A new title"
+description: "The description Date, Open, High"
+Date,Open,High,Low,Last,Close,Total Trade Quantity,Turnover (Lacs)
+2013-11-22,1252.0,454.95,448.2,450.0,450.0,1354405.0,6099.41
+2013-11-21,452.25,457.75,449.1,451.2,451.0,218881.0,992.94

data/spec/lib/quandl/format/dataset/errors_spec.rb

@@ -2,51 +2,14 @@
 require 'spec_helper'
 
 describe Quandl::Format::Dataset do
-  subject{ data }
-
-  context "valid.qdf" do
-    let(:data){ Quandl::Format::Dataset.load( fixtures_data['valid'] ) }
-    
-    it{ should be_a Array }
-    its(:count){ should eq 3 }
-    
-    describe "#first" do
-      subject{ data.first }
-      its(:code){ should eq 'BLAKE_TEST_1' }
-      its(:name){ should eq 'A new title' }
-      its(:description){ should eq 'The description Date, Open, High'}
-      its(:column_names){ should eq ['Date','Open','High','Low','Last','Close','Total Trade Quantity','Turnover (Lacs)']}
-      its(:data){ should eq Quandl::Data.new([['2013-11-22','1252.0','454.95','448.2','450.0','450.0','1354405.0','6099.41'],['2013-11-21','452.25','457.75','449.1','451.2','451.0','218881.0','992.94']]) }
-    end
-  end
-  
-  context "annual.qdf" do
-    let(:data){ Quandl::Format::Dataset.load( fixtures_data['annual'] ) }
-    
-    it{ should be_a Array }
-    its(:count){ should eq 1 }
-    
-    describe "#first" do
-      subject{ data.first }
-      its(:code){ should eq 'ANNUAL_DATA' }
-      its(:name){ should eq 'A new title' }
-      its(:description){ should eq 'Annual Data'}
-      its(:column_names){ should eq ['Date','Open','High']}
-      its(:data){ should eq [
-        [ Date.parse('2013-12-31'), 1252.0, 454.95 ],
-        [ Date.parse('2012-12-31'), 452.25, 457.75 ],
-        [ Date.parse('2011-12-31'), 452.25, 457.75 ],
-        [ Date.parse('2010-12-31'), 452.25, 457.75 ],
-      ]}
-    end
-  end
-
   expected_errors = [
     { file: 'invalid_data',       error: /Date/ },
     { file: 'unknown_attribute',  error: /this_attribute_does_not_exist/ },
     { file: 'mismatched_columns', error: /column_names had 4 columns/ },
     { file: 'mismatched_rows',    error: /had 3 columns/ },
     { file: 'invalid_yaml',       error: /could not find expected ':'/ },
+    { file: 'missing_dashes',     error: /Attribute parse error at line 6 column 1/ },
+    { file: 'missing_dashes',     error: /Data delimiter '-' is missing/ },
   ]
   # run each expectation
   expected_errors.each do |pair|

data/spec/lib/quandl/format/dataset/valid_data_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+
+describe Quandl::Format::Dataset do
+  subject{ data }
+
+  context "valid.qdf" do
+    let(:data){ Quandl::Format::Dataset.load( fixtures_data['valid'] ) }
+    
+    it{ should be_a Array }
+    its(:count){ should eq 3 }
+    
+    describe "#first" do
+      subject{ data.first }
+      its(:code){ should eq 'BLAKE_TEST_1' }
+      its(:name){ should eq 'A new title' }
+      its(:description){ should eq 'The description Date, Open, High'}
+      its(:column_names){ should eq ['Date','Open','High','Low','Last','Close','Total Trade Quantity','Turnover (Lacs)']}
+      its(:data){ should eq Quandl::Data.new([['2013-11-22','1252.0','454.95','448.2','450.0','450.0','1354405.0','6099.41'],['2013-11-21','452.25','457.75','449.1','451.2','451.0','218881.0','992.94']]) }
+    end
+  end
+  
+  context "annual.qdf" do
+    let(:data){ Quandl::Format::Dataset.load( fixtures_data['annual'] ) }
+    
+    it{ should be_a Array }
+    its(:count){ should eq 1 }
+    
+    describe "#first" do
+      subject{ data.first }
+      its(:code){ should eq 'ANNUAL_DATA' }
+      its(:name){ should eq 'A new title' }
+      its(:description){ should eq 'Annual Data'}
+      its(:column_names){ should eq ['Date','Open','High']}
+      its(:data){ should eq [
+        [ Date.parse('2013-12-31'), 1252.0, 454.95 ],
+        [ Date.parse('2012-12-31'), 452.25, 457.75 ],
+        [ Date.parse('2011-12-31'), 452.25, 457.75 ],
+        [ Date.parse('2010-12-31'), 452.25, 457.75 ],
+      ]}
+    end
+  end
+  
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quandl_format
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Blake Hilscher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-09 00:00:00.000000000 Z
+date: 2013-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -169,6 +169,7 @@ files:
 - spec/fixtures/data/invalid_yaml.qdf
 - spec/fixtures/data/mismatched_columns.qdf
 - spec/fixtures/data/mismatched_rows.qdf
+- spec/fixtures/data/missing_dashes.qdf
 - spec/fixtures/data/unknown_attribute.qdf
 - spec/fixtures/data/valid.qdf
 - spec/fixtures/format.rb
@@ -177,6 +178,7 @@ files:
 - spec/lib/quandl/format/dataset/client_spec.rb
 - spec/lib/quandl/format/dataset/errors_spec.rb
 - spec/lib/quandl/format/dataset/load_spec.rb
+- spec/lib/quandl/format/dataset/valid_data_spec.rb
 - spec/lib/quandl/format/dataset_spec.rb
 - spec/spec_helper.rb
 homepage: http://blake.hilscher.ca/
@@ -211,6 +213,7 @@ test_files:
 - spec/fixtures/data/invalid_yaml.qdf
 - spec/fixtures/data/mismatched_columns.qdf
 - spec/fixtures/data/mismatched_rows.qdf
+- spec/fixtures/data/missing_dashes.qdf
 - spec/fixtures/data/unknown_attribute.qdf
 - spec/fixtures/data/valid.qdf
 - spec/fixtures/format.rb
@@ -219,5 +222,6 @@ test_files:
 - spec/lib/quandl/format/dataset/client_spec.rb
 - spec/lib/quandl/format/dataset/errors_spec.rb
 - spec/lib/quandl/format/dataset/load_spec.rb
+- spec/lib/quandl/format/dataset/valid_data_spec.rb
 - spec/lib/quandl/format/dataset_spec.rb
 - spec/spec_helper.rb