quandl_format 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d82a7ccd02fbe5b8359fd3906eb6c7b2598d3d72
-  data.tar.gz: 04d70cbdc92e2bc1be083ed81e4ccc79df33b631
+  metadata.gz: e6a154e15469886cd35f5722374918456c299200
+  data.tar.gz: f5b673bb2619677b9610bd0820feac55c1969bba
 SHA512:
-  metadata.gz: e0260e134f074a1b908a7e6995f54d21aacdffea30ae3faf7da1484d7b74a611d180ad8b399c08c431b3a6a78a2a43d072a3d7d6f7f0eba0ac6d1c13edacb9f6
-  data.tar.gz: d9eaba2e40f27ba8a1897c1afe107bfa983a38e47af7978106f7bdcfd4d0b8281c3b5e7bebe468ec91de9c356a8a8e6bf9f1d0d34e75880fd3cc1730326732dd
+  metadata.gz: 4dd7857d5d4d3b62ef46e2fbd5951f9e4106c7e47b0b24ff20c19caf66e04b2cecae1a4c39ecbb6fa485badb23262e9b761616596047d38781b4efb0fdd0220d
+  data.tar.gz: 2b3b289b1db9c084e4edf2b6d00e320c65c5ce4193e1cd7b286aa1fb6adeb6d5d6fb7a50c3006776783a4f7dd0215665a2463f2b8eb886f0f07d8f241b592a6e

data/Gemfile

@@ -1,2 +1,13 @@
 source "https://rubygems.org"
 gemspec
+
+use_local_gems = ENV['BUNDLE_LOCAL_GEMS'] == "true" && ENV['BUNDLE_LOCAL_DIR']
+local_gem_dir = ENV['BUNDLE_LOCAL_DIR']
+
+if use_local_gems
+  # gem 'her', path: "#{local_gem_dir}/her"
+  gem 'quandl_client', path: "#{local_gem_dir}/quandl/client"
+  gem 'quandl_data', path: "#{local_gem_dir}/quandl/data"
+  gem 'quandl_operation', path: "#{local_gem_dir}/quandl/operation"
+  gem 'quandl_babelfish', path: "#{local_gem_dir}/quandl/babelfish"
+end

data/examples/load.rb

@@ -0,0 +1,9 @@
+require 'pry'
+require 'quandl/format'
+
+Quandl::Logger.use Quandl::Logger::Outputs
+
+Quandl::Format::Dataset.load_from_file('spec/fixtures/data/invalid_data.qdf')
+Quandl::Format::Dataset.load_from_file('spec/fixtures/data/invalid_yaml.qdf')
+Quandl::Format::Dataset.load_from_file('spec/fixtures/data/mismatched_columns.qdf')
+Quandl::Format::Dataset.load_from_file('spec/fixtures/data/mismatched_rows.qdf')

data/lib/quandl/format/dataset/attributes.rb

@@ -82,17 +82,6 @@ module Attributes
     self.class.attribute_names.inject({}){|m,k| m[k] = self.send(k); m }
   end
 
-  def inspect
-    attrs = attributes.collect do |key, value|
-      if value.is_a?(String)
-        value = "#{value[0..20]}..." if value.length > 20
-        value = "'#{value}'"
-      end
-      "#{key}: #{value}"
-    end
-    %Q{<##{self.class.name} #{attrs.join(', ')}>}
-  end
-
   protected
   
   def data_rows_should_have_equal_columns!

data/lib/quandl/format/dataset/load.rb

@@ -20,7 +20,10 @@ class Quandl::Format::Dataset::Load
     def parse_string(input)
       nodes = []
       section_type = :data
+      line_index = 0
       input.each_line do |rline|
+        # track current line index
+        line_index += 1
         # strip whitespace
         line = rline.strip.rstrip
         # ignore comments and blank lines
@@ -30,7 +33,7 @@ class Quandl::Format::Dataset::Load
         if line =~ attribute_format
           # if we are leaving the data section
           # then this is the start of a new node
-          nodes << { attributes: '', data: '' } if section_type == :data
+          nodes << { attributes: '', data: '', line: line_index } if section_type == :data
           # update the section to attributes
           section_type = :attributes
           
@@ -49,35 +52,50 @@ class Quandl::Format::Dataset::Load
     end
     
     def parse_yaml_and_csv(nodes)
-      nodes.collect do |node|
+      output = []
+      nodes.each do |node|
         # parse attrs as yaml
-        node[:attributes] = YAML.load( node[:attributes] ).symbolize_keys!
+        node[:attributes] = parse_yaml_attributes(node)
+        # we cant continue unless attributes are present
+        next if node[:attributes].blank?
         # parse data as csv
         node[:attributes][:data] = CSV.parse(node[:data])
-        node
+        # onwards
+        output << node
       end
+      output
     end
     
     def nodes_to_datasets(nodes)
       datasets = []
-      nodes.each_with_index do |node, index|
-        dataset = node_to_dataset(node, index)
+      nodes.each do |node|
+        dataset = node_to_dataset(node)
         datasets << dataset if dataset
       end
       datasets
     end
     
-    def node_to_dataset(node, index)
+    def parse_yaml_attributes(node)
+      YAML.load( node[:attributes] ).symbolize_keys!
+    rescue => e
+      message = "Error: Dataset starting at line #{node[:line]}\n"
+      message += "#{$!}\n"
+      message += "--"
+      Quandl::Logger.error(message)
+      nil
+    end
+    
+    def node_to_dataset(node)
       Quandl::Format::Dataset.new( node[:attributes] )
-    rescue => e# Quandl::Format::Errors::UnknownAttribute => e
-      message = "Error: Dataset #{index + 1}\n"
+    rescue => e
+      message = ''
       message += node[:attributes][:source_code] + '/' if node[:attributes][:source_code].present?
-      message += node[:attributes][:code] + "\n"
+      message += node[:attributes][:code] + ' '
+      message += "error around line #{node[:line]} \n"
       message += "#{$!}\n"
       message += "--"
-      Quandl::Logger.error(message, e)
+      Quandl::Logger.error(message)
       nil
-      # raise $!, message, $!.backtrace
     end
     
     def attribute_format

data/lib/quandl/format/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Format
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end
 end

data/spec/fixtures/data/invalid_yaml.qdf

@@ -0,0 +1,20 @@
+name: "A new title"
+code "test"
+description: "The description Date, Open, High"
+private: false
+-
+2013-11-22,1252.0,454.95,448.2,450.0,450.0,1354405.0,6099.41
+
+name: "A new title"
+code: "test"
+description: "The description Date, Open, High"
+private: false
+-
+2013-11-22,1252.0,454.95,448.2,450.0,450.0,1354405.0,6099.41
+
+name: "A new title"
+description: "The description Date, Open, High"
+private: false
+asomjkl
+-
+2013-11-22,1252.0,454.95,448.2,450.0,450.0,1354405.0,6099.41

data/spec/fixtures/format.rb

@@ -19,8 +19,8 @@ description: |-
   This is the second line.
 -
 Date, Value, High, Low
-2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
-2013-11-19,10.039388096885814,,14.09718770934256
+2013-11-20,9.99,11,14
+2013-11-19,10.03,,14.09
 
 # Second dataset
 code:           DATASET_CODE_2
@@ -29,9 +29,9 @@ name:           Test Dataset Name 2
 description:    Here is a description with multiple lines.
 -
 Date, Value, High, Low
-2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
-2013-11-19,10.039388096885814,,14.09718770934256
-2013-11-18,11.039388096885814,,15.09718770934256
+2013-11-20,9.99,11.0,14.0
+2013-11-19,10.03,,14.09
+2013-11-18,11.03,,15.09
 }
 end
 
@@ -44,7 +44,7 @@ def qdf_attributes
     column_names:   ['Date', 'Value', 'High', 'Low'],
     private:        false,
     reference_url:    'http://test.com/',
-    data:           Quandl::Data.new([["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"],["2013-11-19", "10.039388096885814", nil, "14.09718770934256"]]),
+    data:           Quandl::Data.new([["2013-11-20", "9.99", "11.0", "14.0"],["2013-11-19", "10.03", nil, "14.09"]]),
   }
 end
 
@@ -59,7 +59,7 @@ private: false
 reference_url: http://test.com/
 -
 Date,Value,High,Low
-2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
-2013-11-19,10.039388096885814,,14.09718770934256
+2013-11-20,9.99,11.0,14.0
+2013-11-19,10.03,,14.09
 }
 end

data/spec/lib/quandl/format/dataset/errors_spec.rb

@@ -18,25 +18,20 @@ describe Quandl::Format::Dataset do
       its(:data){ should eq Quandl::Data.new([['2013-11-22','1252.0','454.95','448.2','450.0','450.0','1354405.0','6099.41'],['2013-11-21','452.25','457.75','449.1','451.2','451.0','218881.0','992.94']]) }
     end
   end
-  
-  context "invalid_data.qdf" do
-    let(:data){ Quandl::Format::Dataset.load( fixtures_data['invalid_data'] ) }
-    it{ expect{data}.to raise_error Quandl::Operation::Errors::UnknownDateFormat, /Date/ }
-  end
-  
-  context "unknown_attribute.qdf" do
-    let(:data){ Quandl::Format::Dataset.load( fixtures_data['unknown_attribute'] ) }
-    it{ expect{data}.to raise_error Quandl::Format::Errors::UnknownAttribute, /this_attribute_does_not_exist/ }
-  end
-  
-  context "mismatched_columns.qdf" do
-    let(:data){ Quandl::Format::Dataset.load( fixtures_data['mismatched_columns'] ) }
-    it{ expect{data}.to raise_error Quandl::Format::Errors::ColumnCountMismatch, /column_names had 4 columns/ }
-  end
-  
-  context "mismatched_rows.qdf" do
-    let(:data){ Quandl::Format::Dataset.load( fixtures_data['mismatched_rows'] ) }
-    it{ expect{data}.to raise_error Quandl::Format::Errors::ColumnCountMismatch, /had 3 columns/ }
+
+  expected_errors = [
+    { file: 'invalid_data',       error: /Date/ },
+    { file: 'unknown_attribute',  error: /this_attribute_does_not_exist/ },
+    { file: 'mismatched_columns', error: /column_names had 4 columns/ },
+    { file: 'mismatched_rows',    error: /had 3 columns/ },
+    { file: 'invalid_yaml',       error: /could not find expected ':'/ },
+  ]
+  # run each expectation
+  expected_errors.each do |pair|
+    it "#{pair[:file]}.qdf should error with #{pair[:error]}" do
+      Quandl::Logger.should_receive(:error).at_least(:once).with(pair[:error])
+      Quandl::Format::Dataset.load( fixtures_data[pair[:file]] )
+    end
   end
   
 end

data/spec/lib/quandl/format/dataset/load_spec.rb

@@ -26,7 +26,7 @@ describe Quandl::Format::Dataset::Load do
       its(:name){ should eq 'Oil India Limited' }
       its(:description){ should eq "Here is a description with multiple lines.\nThis is the second line." }
       its(:column_names){ should eq ['Date', 'Value', 'High', 'Low'] }
-      its(:data){ should eq Quandl::Data.new([["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"],["2013-11-19", "10.039388096885814", nil, "14.09718770934256"]]) }
+      its(:data){ should eq Quandl::Data.new([["2013-11-20", "9.99", "11.0", "14.0"],["2013-11-19", "10.03", nil, "14.09"]]) }
     end
   end
   

data/spec/lib/quandl/format/dataset_spec.rb

@@ -12,7 +12,7 @@ describe Quandl::Format::Dataset do
   its(:name){ should eq 'Test Dataset Name 2' }
   its(:description){ should eq "Here is a description with multiple lines.\n This is the second line." }
   its(:column_names){ should eq ['Date', 'Value', 'High', 'Low'] }
-  its(:data){ should eq Quandl::Data.new([["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"], ["2013-11-19", "10.039388096885814", nil, "14.09718770934256"]]) }
+  its(:data){ should eq Quandl::Data.new([["2013-11-20", "9.99", "11.0", "14.0"], ["2013-11-19", "10.03", nil, "14.09"]]) }
   
   its(:attributes){ should eq attributes }
   

data/spec/spec_helper.rb

@@ -14,4 +14,4 @@ require 'pry'
 
 # Replace Quandl::Logger with Spec::Logger that will raise errors sent to #error
 # This allows us to easily test error assertions in spec/lib/quandl/format/errors_spec.rb
-Quandl::Logger.use(Spec::Logger)
+Quandl::Logger.use(Quandl::Logger::Outputs)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quandl_format
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Blake Hilscher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-05 00:00:00.000000000 Z
+date: 2013-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -136,6 +136,7 @@ files:
 - README.md
 - Rakefile
 - UPGRADE.md
+- examples/load.rb
 - lib/quandl/client/dataset/to_qdf.rb
 - lib/quandl/format.rb
 - lib/quandl/format/dataset.rb
@@ -149,6 +150,7 @@ files:
 - spec/config/client.rb
 - spec/config/logger.rb
 - spec/fixtures/data/invalid_data.qdf
+- spec/fixtures/data/invalid_yaml.qdf
 - spec/fixtures/data/mismatched_columns.qdf
 - spec/fixtures/data/mismatched_rows.qdf
 - spec/fixtures/data/unknown_attribute.qdf
@@ -189,6 +191,7 @@ test_files:
 - spec/config/client.rb
 - spec/config/logger.rb
 - spec/fixtures/data/invalid_data.qdf
+- spec/fixtures/data/invalid_yaml.qdf
 - spec/fixtures/data/mismatched_columns.qdf
 - spec/fixtures/data/mismatched_rows.qdf
 - spec/fixtures/data/unknown_attribute.qdf