quandl_format 0.2.7 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b474c3187e8739850c58e91360e3ab2b59340f2d
-  data.tar.gz: e9b54e19bc1ec87383303bc285d60b42ca421f05
+  metadata.gz: 2c2355e2207291cd09c76a495b2565c28fefdbd0
+  data.tar.gz: 5699891dd773b8d58e4824d4a7411b39db054e92
 SHA512:
-  metadata.gz: be6a15eff00055881aaff30529320a3c34c6337990ed0bf13e03cc85307bb3a52a82fddd8f3d7a84c8030dd1946611afe714df6a130f7614695b96edc0fec938
-  data.tar.gz: 947092cce001ee7cfa1dd4c3c3505dd5a82007bf45150a134e015b95cc44172f661793b418cacbfb8364e4b62d2d9094eca1e4494a2b16e156763ef5b5191f4d
+  metadata.gz: 6426c47bc68a9891942a529ba2220a3360de14031e68f7b8d117a5faaf01cad4ba8e30c406b6c101778c09a6a33dddb7f04037ef26b5ceff4a4807f4b6e35606
+  data.tar.gz: a0db81d386ea365f37d2f64007c04f37ec6e4cb7f0fdb577905ae6824eaeded19545c675bf7ac20f52d2574d989e0613a7a7bcd8548598443a2e76ed0dc584e8

data/UPGRADE.md

@@ -1,3 +1,9 @@
+## 0.2.8
+
+* QUGC-57 validation at an inappropriate time; move validations to Quandl::Client
+* more accurate error messages and line numbers
+
+
 ## 0.2.7
 
 * parsing junk string should fail gracefully

data/lib/quandl/format/dataset/attributes.rb

@@ -62,8 +62,6 @@ module Attributes
     rows = rows.to_table if rows.respond_to?(:to_table)
     @data = Quandl::Data.new(rows).to_date
     self.column_names = @data.headers if @data.headers.present?
-    data_row_count_should_match_column_count!
-    data_rows_should_have_equal_columns!
     @data
   end
   
@@ -86,38 +84,9 @@ module Attributes
   def attributes
     self.class.attribute_names.inject({}){|m,k| m[k] = self.send(k) unless self.send(k).nil?; m }
   end
-
-  protected
-  
-  def data_rows_should_have_equal_columns!
-    return if data.blank?
-    row_count = data[0].count
-    data.each_with_index do |row, index|
-      raise_row_column_mismatch!(row, index) unless row.count == row_count
-    end
-  end
-  
-  def data_row_count_should_match_column_count!
-    return if column_names.blank?
-    column_count = column_names.count
-    data.each_with_index do |row, index|
-      raise_column_count_mismatch!(row, index) unless row.count == column_count
-    end
-  end
   
-
   private
   
-  def raise_row_column_mismatch!(row, index)
-    m = "Unexpected number of points in this row '#{row}'. Expected #{data[0].count} but found #{row.count} based on #{data[0]}"
-    raise Quandl::Error::ColumnCountMismatch.new( row: row, line: index, context: 'data_rows_should_have_equal_columns!' ), m
-  end
-  
-  def raise_column_count_mismatch!(row, index)
-    m = "Unexpected number of points in this row '#{row}'. Expected #{column_names.count} but found #{row.count} based on #{column_names}"
-    raise Quandl::Error::ColumnCountMismatch.new( row: row, line: index+1, context: 'data_row_count_should_match_column_count!' ), m
-  end
-  
   def raise_unknown_attribute_error!(key)
     m = "Unknown Field '#{key}' valid fields are: #{self.class.meta_attribute_names.join(', ')}"
     raise Quandl::Error::UnknownAttribute, m

data/lib/quandl/format/dataset/load.rb

@@ -19,7 +19,7 @@ class Quandl::Format::Dataset::Load
         # process line
         node = process_line(line, node, &block)
       end
-      process_tail(node, &block)      
+      process_tail(node, &block)
     end
     
     def file(path)
@@ -43,7 +43,7 @@ class Quandl::Format::Dataset::Load
     end
     
     def new_node(line=0)
-      { line: line, section: :attributes, data: '', attributes: '', data_line: 0 }
+      { line: line, section: :attributes, data: '', attributes: '', data_line: 0, offset: line==0 ? 0 : line-1 }
     end
     
     def process_tail(node, &block)
@@ -58,7 +58,10 @@ class Quandl::Format::Dataset::Load
       # strip whitespace
       line = rline.strip.rstrip
       # skip comments and blank lines
-      return node if line[0] == SYNTAX[:comment] || line.blank?
+      if line[0] == SYNTAX[:comment] || line.blank?
+        node[:attributes] += "\n" if node[:section] == :attributes
+        return node
+      end
       # looking at an attribute?
       if line =~ SYNTAX[:attribute]
         # exiting data section?
@@ -111,7 +114,9 @@ class Quandl::Format::Dataset::Load
     
     def parse_yaml_attributes(node)
       attrs = {}
-      YAML.load( node[:attributes] ).symbolize_keys!.each do |key, value|
+      attributes = YAML.load( node[:attributes] )
+      raise 'Unparsable input'  unless attributes.is_a? Hash
+      attributes.symbolize_keys!.each do |key, value|
         attrs[key.to_s.downcase.to_sym] = value
       end
       attrs
@@ -131,9 +136,21 @@ class Quandl::Format::Dataset::Load
     
     def log_yaml_parse_error(node, err)
       message = ""
-      message += "Attribute parse error at line #{ node[:line] + err.line } column #{err.column}. #{err.problem} (#{err.class})\n" if node.has_key?(:line) && err.respond_to?(:line)
-      message += "Did you forget to delimit the meta data section from the data section with a one or more dashes ('#{SYNTAX[:data]}')?\n" unless node[:attributes] =~ /^-/
-      message += "Encountered error while parsing: \n  " + node[:attributes].split("\n")[err.line - 1].to_s + "\n" if err.respond_to?(:line)
+      if err.message == 'Unparsable input'
+        message = "Input data is unparsable.  Are you missing a colon (:) or a space after a colon?\n"
+      elsif err.is_a?(Psych::SyntaxError)
+        if err.problem =~ /mapping values are not allowed in this context/
+          message = "Syntax error before line #{1+node[:offset] + err.line}.  Are you missing a colon (:) or a space after a colon?\n"
+        else
+          message += "Error parsing metadata. #{err.problem.capitalize} on line #{node[:offset] + err.line}\n"
+          if err.problem =~ /expected ':'/
+            message += "Did you forget to delimit the meta data section from the data section with a one or more dashes ('#{SYNTAX[:data]}')?\n"
+          end
+        end
+      else
+        message += "Attribute parse error at line #{ node[:line] + err.line } column #{err.column}. #{err.problem} (#{err.class})\n" if node.has_key?(:line) && err.respond_to?(:line)
+        message += "Encountered error while parsing: \n  " + node[:attributes].split("\n")[err.line - 1].to_s + "\n" if err.respond_to?(:line)
+      end
       message += "--"
       Quandl::Logger.error(message)
     end

data/lib/quandl/format/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Format
-  VERSION = "0.2.7"
+  VERSION = "0.2.8"
 end
 end

data/spec/fixtures/data/missing_colon.qdf

@@ -0,0 +1,2 @@
+code foo
+name: hello

data/spec/fixtures/data/missing_colon2.qdf

@@ -0,0 +1,3 @@
+code: FOO
+name: Bar
+description forgot the colon, damn.

data/spec/fixtures/data/missing_dashes.qdf

@@ -2,6 +2,7 @@ code: "BLAKE_TEST_1"
 name: "A new title"
 description: "The description Date, Open, High"
 private: false
+-
 Date,Open,High,Low,Last,Close,Total Trade Quantity,Turnover (Lacs)
 2013-11-22,1252.0,454.95,448.2,450.0,450.0,1354405.0,6099.41
 2013-11-21,452.25,457.75,449.1,451.2,451.0,218881.0,992.94
@@ -9,13 +10,23 @@ Date,Open,High,Low,Last,Close,Total Trade Quantity,Turnover (Lacs)
 code: "BLAKE_TEST_2"
 name: "A new title"
 description: "The description Date, Open, High"
+-
 Date,Open,High,Low,Last,Close,Total Trade Quantity,Turnover (Lacs)
 2013-11-22,1252.0,454.95,448.2,450.0,450.0,1354405.0,6099.41
 2013-11-21,452.25,457.75,449.1,451.2,451.0,218881.0,992.94
 
+# Missing Dash in this one
 code: "BLAKE_TEST_3"
 name: "A new title"
 description: "The description Date, Open, High"
 Date,Open,High,Low,Last,Close,Total Trade Quantity,Turnover (Lacs)
 2013-11-22,1252.0,454.95,448.2,450.0,450.0,1354405.0,6099.41
+2013-11-21,452.25,457.75,449.1,451.2,451.0,218881.0,992.94
+
+code: "BLAKE_TEST_4"
+name: "A new title"
+description: "The description Date, Open, High"
+-
+Date,Open,High,Low,Last,Close,Total Trade Quantity,Turnover (Lacs)
+2013-11-22,1252.0,454.95,448.2,450.0,450.0,1354405.0,6099.41
 2013-11-21,452.25,457.75,449.1,451.2,451.0,218881.0,992.94

data/spec/fixtures/data/missing_space.qdf

@@ -0,0 +1 @@
+code:FOO

data/spec/lib/quandl/format/dataset/load/errors_spec.rb

@@ -12,10 +12,11 @@ describe Quandl::Format::Dataset do
   
   it_should_expect_error 'invalid_data',       /Date/
   it_should_expect_error 'unknown_attribute',  /this_attribute_does_not_exist/
-  it_should_expect_error 'mismatched_columns', /Expected 4 but found 5/
-  it_should_expect_error 'mismatched_rows',    /Expected 3 but found 4/
-  it_should_expect_error 'invalid_yaml',       /could not find expected ':'/
-  it_should_expect_error 'missing_dashes',     /Attribute parse error at line 28 column 1/
+  it_should_expect_error 'invalid_yaml',       /Could not find expected ':'/
+  it_should_expect_error 'missing_dashes',     /Could not find expected ':' on line 22/
   it_should_expect_error 'missing_dashes',     /Did you forget to delimit the meta data section/
+  it_should_expect_error 'missing_colon',     /Are you missing a colon/
+  it_should_expect_error 'missing_colon2',     /Could not find expected ':' on line 3/
+  it_should_expect_error 'missing_space',     /Are you missing a colon/
   
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quandl_format
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.2.8
 platform: ruby
 authors:
 - Blake Hilscher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-03 00:00:00.000000000 Z
+date: 2014-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: quandl_client
@@ -157,7 +157,10 @@ files:
 - spec/fixtures/data/metadata_only.qdf
 - spec/fixtures/data/mismatched_columns.qdf
 - spec/fixtures/data/mismatched_rows.qdf
+- spec/fixtures/data/missing_colon.qdf
+- spec/fixtures/data/missing_colon2.qdf
 - spec/fixtures/data/missing_dashes.qdf
+- spec/fixtures/data/missing_space.qdf
 - spec/fixtures/data/tabular.qdf
 - spec/fixtures/data/unknown_attribute.qdf
 - spec/fixtures/data/valid.qdf
@@ -204,7 +207,10 @@ test_files:
 - spec/fixtures/data/metadata_only.qdf
 - spec/fixtures/data/mismatched_columns.qdf
 - spec/fixtures/data/mismatched_rows.qdf
+- spec/fixtures/data/missing_colon.qdf
+- spec/fixtures/data/missing_colon2.qdf
 - spec/fixtures/data/missing_dashes.qdf
+- spec/fixtures/data/missing_space.qdf
 - spec/fixtures/data/tabular.qdf
 - spec/fixtures/data/unknown_attribute.qdf
 - spec/fixtures/data/valid.qdf