quandl_data 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca5bc03c4ec953809ecabafb2e637410965b1404
-  data.tar.gz: 6e44598ca8bc5256d7a002c7f84bde9e6de5c0ba
+  metadata.gz: e811fb881ccd062b98502cddce1c7a27d5f53ac2
+  data.tar.gz: 5f84494fdae5960ebb598c032da19cf9ce342b3d
 SHA512:
-  metadata.gz: 2bc2f15ee8b8a799b0dc95fb6d5be8e4b4aa98aad1e2532cf59aea3d298a3c0d82a6470703cde24dc42b36eac3ccbcbaf053062720cd3012088efb964d731986
-  data.tar.gz: e654676035a82af7479e53cd706d8f4f0d2a93275beb58d79ae1bbc7adbc11ca2915c5a79c825ede14690122afb57004acc99ba0eeceb4fec451276eeb50686a
+  metadata.gz: 514a0425af0aa3af800bb8a0c2f06299bc9a2bce9b825cccc57ad84cd8469107d2da25a5a4be4900bcf6c4d78958edb34a5d496d528467f64489a8f5e8e9d8ac
+  data.tar.gz: 91ff9683ef3c984c2e7e679178b04848699ede82b7e54b6f36b883d19823697888258d2f9b67fdf2803e72bfbda6ec88c7eab526c39346c6f631321059866f42

data/Gemfile

@@ -6,4 +6,5 @@ local_gem_dir = ENV['BUNDLE_LOCAL_DIR']
 
 if use_local_gems
   gem 'quandl_operation', path: "#{local_gem_dir}/quandl/operation"
+  gem 'quandl_babelfish', path: "#{local_gem_dir}/quandl/babelfish"
 end

data/UPGRADE.md

@@ -1,3 +1,16 @@
+## 1.3.0
+
+* string_to_csv will not force values to float
+* add #row to Operations
+* add cleaned? cleaned! for tracking whether or not a Quandl::Data instance has already been cleaned
+* replace parse with babelfish#clean
+* add Quandl::Babelfish.clean to replace Quandl::Data::Format.parse
+* bump Quandl::Operation to 0.3.0 for Quandl::Operation::Sort
+* refactor sorting into Quandl::Operation::Sort
+* import Quandl::Data::Format from Quandl::Operation
+* remove data parse
+
+
 ## 1.2.1
 
 * add Data#to_date_str that returns data as "yyyy-mm-dd"

data/lib/quandl/data/babelfisher.rb

@@ -0,0 +1,56 @@
+module Quandl
+class Data
+module Babelfisher
+  
+  extend ActiveSupport::Concern
+  
+  def headers=(value)
+    @headers = value
+  end
+  def headers
+    @headers ||= data_array.respond_to?(:headers) ? data_array.headers : nil
+  end
+  def headers?
+    headers.present?
+  end
+  
+  def cleaned
+    @cleaned
+  end
+  def cleaned=(value)
+    @cleaned = (value == true)
+  end
+  def cleaned?
+    cleaned == true
+  end
+
+  protected
+  
+  def clean(data)
+    # skip cleaning if already clean
+    return data if data.kind_of?(Array) && cleaned?
+    # Quandl::Data is already clean, but to avoid errors extract internal array
+    return data.data_array if data.kind_of?(Quandl::Data)
+    # Quandl::Babelfish::Data is already clean
+    return data if data.kind_of?(Quandl::Babelfish::Data)
+    # Return empty array if given empty string, nil, etc.
+    return [] if data.blank?
+    # String needs conversion to array
+    data = Format.csv_to_array( data ) if data.kind_of?(String)
+    # Hash needs conversion to array
+    data = Format.hash_to_array( data ) if data.kind_of?(Hash)
+    # Array needs cleaning
+    data = Quandl::Babelfish.clean( data )
+    # mark data as clean
+    cleaned!
+    # return data
+    data
+  end
+
+  def cleaned!
+    self.cleaned = true
+  end
+  
+end
+end
+end

data/lib/quandl/data/enumerator.rb

@@ -5,7 +5,7 @@ module Quandl::Data::Enumerator
     delegate *Array.forwardable_methods, to: :data_array
   
     delegate :to_json, :as_json, to: :data_array
-  
+    
     attr_accessor :pristine_data
   end
   
@@ -15,11 +15,11 @@ module Quandl::Data::Enumerator
   end
   
   def data_array
-    @data_array ||= parse( pristine_data )
+    @data_array ||= clean( pristine_data )
   end
 
   def data_array=(data)
-    @data_array = parse(data)
+    @data_array = clean(data)
   end
   
   def data_array?
@@ -42,12 +42,4 @@ module Quandl::Data::Enumerator
     end
   end
   
-  protected
-  
-  def parse(data)
-    data = data.to_a if data.respond_to?(:to_a) && data.is_a?(Quandl::Data)
-    data = Quandl::Operation::Parse.perform( data )
-    data
-  end
-  
 end

data/lib/quandl/data/format.rb

@@ -0,0 +1,217 @@
+module Quandl
+class Data
+class Format
+  
+  class << self
+  
+    def parse(data)
+      return [] if data.blank?
+      data = hash_to_array(data)
+      data = csv_to_array(data)
+      data = values_to_float(data)
+      data = to_date(data)
+      data.dup
+    end
+    
+    def hash_to_array(data)
+      data = data.collect{|k,v| [k] + v } if data.kind_of?(Hash)
+      data
+    end
+  
+    def csv_to_array(data)
+      if data.is_a?(String)
+        data = data.gsub('\n', "\n")
+        data = CSV.parse( data )
+      end
+      data
+    end
+    
+    def to_date(data)
+      return data if data_missing_rows?(data)
+      # guess the current date format
+      format = date_format?(data)
+      # convert dates to Date
+      case format
+      when :date            then return data
+      when :date_string     then return date_strings_to_date( data )
+      when :jd, :jd_string  then return jds_to_date( data )
+      when :unknown         then raise_date_format_error!( data[0] )
+      end
+      # return data
+      data
+    end
+    
+    def to_jd(data)
+      return data if data_missing_rows?(data)
+      # guess the current date format
+      format = date_format?(data)
+      # convert dates to Date
+      case format
+      when :jd          then return data
+      when :jd_string   then return jd_strings_to_jd( data )
+      when :date        then return dates_to_jd( data )
+      when :date_string then return date_strings_to_jd( data )
+      when :unknown     then raise_date_format_error!( data[0] )
+      end
+      # return data
+      data
+    end
+    
+    def date_format?(data)
+      value = data[0][0]
+      # julian date?
+      return :date          if value.is_a?(Date)
+      return :jd            if value.is_a?(Integer)
+      return :date_string   if value.is_a?(String) && value =~ /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/
+      return :jd_string     if value.kind_of?(String) && value.numeric?
+      return :unknown
+    end
+    
+    def dates_to_jd(data)
+      return data if data_missing_rows?(data) || data[0][0].is_a?(Integer)
+      # dont alter by reference
+      result = []
+      # for each row
+      data.each_with_index do |row, index|
+        # copy
+        row = row.dup
+        # string to date
+        date = row[0]
+        date = Date.parse(row[0]) if row[0].is_a?(String)
+        # date to julian
+        row[0] = date.jd if date.respond_to?(:jd)
+        # save result
+        result[index] = row
+      end
+      # all done
+      result
+    end
+    
+    
+    ###################
+    # DATES TO JULIAN #
+    ###################
+
+    def jd_strings_to_jd(data)
+      # skip when already formatted correctly
+      return data if data_missing_rows?(data) || data[0][0].is_a?(Integer)
+      # otherwise cast string jds to int
+      output = []
+      data.each_with_index do |row, index|
+        output << parse_jd_string(row) rescue raise_date_format_error!( row, index, :jd_strings_to_jd )
+      end
+      output
+    end
+    
+    def date_strings_to_jd(data)
+      # skip when already formatted correctly
+      return data if data_missing_rows?(data) || data[0][0].is_a?(Date)
+      # otherwise cast string jds to int
+      output = []
+      data.each_with_index do |row, index|
+        row = parse_date_string(row) rescue raise_date_format_error!( row, index, :date_strings_to_jd )
+        row[0] = row[0].jd
+        output << row
+      end
+      output
+    end
+
+    
+    #################
+    # DATES TO DATE #
+    #################
+
+    def jds_to_date(data)
+      return data if data_missing_rows?(data) || data[0][0].is_a?(Date)
+      output = []
+      data.each_with_index do |row, index|
+        output << parse_jd(row) rescue raise_date_format_error!( row, index, :jds_to_date )
+      end
+      output
+    end
+  
+    def date_strings_to_date(data)
+      # skip when already formatted correctly
+      return data if data_missing_rows?(data) || data[0][0].is_a?(Date)
+      # otherwise cast string jds to int
+      output = []
+      data.each_with_index do |row, index|
+        output << parse_date_string(row) rescue raise_date_format_error!( row, index, :date_strings_to_date )
+      end
+      output
+    end
+    
+    
+    ###################
+    # VALUES TO FLOAT #
+    ###################
+
+    def values_to_float(data)
+      # skip unless value is a string
+      return data if data_missing_rows?(data) || data[0][1].is_a?(Float)
+      # cast values to float
+      data.collect do |row|
+        new_row = [row[0]]
+        row[1..-1].each_with_index do |value, index|
+          value = value.to_f if value.is_a?(String)
+          new_row[index + 1] = value
+        end
+        new_row
+      end
+    end
+    
+    
+    def data_missing_rows?(data)
+      !data_has_rows?(data)
+    end
+    
+    def data_has_rows?(data)
+      data.is_a?(Array) && data[0].is_a?(Array) && data[0][0].present?
+    end
+    
+    
+    protected
+    
+    def parse_jd(row)
+      # parse jd_string
+      row = parse_jd_string(row)
+      # jd to date
+      row[0] = Date.jd( row[0] )
+      # onwards
+      row
+    end
+    
+    def parse_jd_string(row)
+      row = row.dup
+      row[0] = row[0].to_i
+      # dont allow dates that are before 0000
+      raise Quandl::Errors::UnknownDateFormat if row[0] <= 1721058
+      row
+    end
+    
+    def parse_date_string(row)
+      row = row.dup
+      # extract date
+      date = row[0]
+      # split date into parts
+      date_values = date.split('-').collect(&:to_i)
+      # ensure date is valid
+      raise Quandl::Errors::UnknownDateFormat unless date_values.count == 3
+      # add to row
+      row[0] = Date.new( *date_values )
+      row
+    end
+    
+    
+    private
+    
+    def raise_date_format_error!(row, index = 0, type = :none)
+      message = "UnknownDateFormat: '#{row[0]}', index: data[#{index}][0], strategy: '#{type}', row: #{row}"
+      raise Quandl::Errors::UnknownDateFormat, message
+    end
+    
+  end
+  
+end
+end
+end

data/lib/quandl/data/operations.rb

@@ -1,10 +1,18 @@
-module Quandl::Data::Operations
+module Quandl
+class Data
+module Operations
   
   extend ActiveSupport::Concern
-  include Quandl::Operation
 
   module ClassMethods
     
+    def new_with_jd(*args)
+      # force data to date
+      args[0] = Format.to_date( args[0] )
+      # onwards
+      self.new( *args )
+    end
+    
     def forwardable_methods
       @forwardable_methods ||= (operations + operations.collect{|o| "#{o}!" if method_defined?("#{o}!") }).compact
     end
@@ -31,11 +39,11 @@ module Quandl::Data::Operations
   end
   
   def to_jd!
-    @data_array = Parse.to_jd( data_array ); self
+    @data_array = Quandl::Data::Format.to_jd( data_array ); self
   end
   
   def to_date!
-    @data_array = Parse.to_date( data_array ); self
+    @data_array = Quandl::Data::Format.to_date( data_array ); self
   end
   
   def to_date_str!
@@ -73,11 +81,18 @@ module Quandl::Data::Operations
   end
   
   def sort_ascending!
-    @data_array = Parse.sort_asc( data_array ); self
+    @data_array = Quandl::Operation::Sort.asc( data_array ); self
   end
 
   def sort_descending!
-    @data_array = Parse.sort_desc( data_array ); self
+    @data_array = Quandl::Operation::Sort.desc( data_array ); self
+  end
+  
+  def row(*args)
+    return @row if args[0].nil?
+    @row = args[0]
+    @data_array = [data_array[ args[0] ]]
+    self
   end
   
   def transform(*args)
@@ -86,9 +101,10 @@ module Quandl::Data::Operations
     self
   end
   def transform=(value)
-    return false unless Transform.valid?(value)
+    return false unless Quandl::Operation::Transform.valid?(value)
     @transform = value
-    @data_array = Transform.perform( data_array, value )
+    @data_array = Quandl::Operation::Transform.perform( data_array, value )
+    @data_array
   end
 
   def collapse(*args)
@@ -97,21 +113,21 @@ module Quandl::Data::Operations
     self
   end
   def collapse=(collapse)
-    return false unless Collapse.valid?(collapse)
+    return false unless Quandl::Operation::Collapse.valid?(collapse)
     @collapse = collapse
     @frequency = collapse
-    @data_array = Collapse.perform( data_array, collapse )
+    @data_array = Quandl::Operation::Collapse.perform( data_array, collapse )
   end
   
   def frequency
-    @frequency ||= Collapse.frequency?( data_array )
+    @frequency ||= Quandl::Operation::Collapse.frequency?( data_array )
   end
   def frequency=(value)
     @frequency = value.to_sym if value.present?
   end
   
   def clone
-    Quandl::Data.new( data_array.dup )
+    self.class.new( data_array.dup, headers: headers, cleaned: cleaned )
   end
   
   def to_jd
@@ -138,4 +154,7 @@ module Quandl::Data::Operations
   def sort_descending
     clone.sort_descending!
   end
+  
+end
+end
 end

data/lib/quandl/data/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 class Data
-  VERSION = "1.2.1"
+  VERSION = "1.3.0"
 end
 end

data/lib/quandl/data.rb

@@ -6,13 +6,21 @@ require "active_support/core_ext/hash"
 require "active_support/core_ext/object"
 
 require "quandl/operation"
+require "quandl/babelfish"
 
+require 'quandl/errors/unknown_date_format'
+
+require 'quandl/data/babelfisher'
 require 'quandl/data/enumerator'
 require 'quandl/data/operations'
+require 'quandl/data/format'
 require 'quandl/data/logging'
 
-class Quandl::Data
+module Quandl
+class Data
+  include Babelfisher
   include Enumerator
   include Operations
   include Logging if defined?(QUANDL_LOGGER) && QUANDL_LOGGER == true
-end
+end
+end

data/lib/quandl/errors/unknown_date_format.rb

@@ -0,0 +1,5 @@
+module Quandl
+module Errors
+  class UnknownDateFormat < StandardError; end
+end
+end

data/quandl_data.gemspec

@@ -17,6 +17,9 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
+  s.add_runtime_dependency "quandl_operation", "~> 0.3.1"
+  s.add_runtime_dependency "quandl_babelfish", "~> 0.0"
+  
   s.add_development_dependency "rake", "~> 10.0"
   s.add_development_dependency "rspec", "~> 2.13"
   s.add_development_dependency "fivemat", "~> 1.2"
@@ -24,7 +27,4 @@ Gem::Specification.new do |s|
   s.add_development_dependency "simplecov"
   s.add_development_dependency "guard"
   s.add_development_dependency "guard-rspec"
-
-  s.add_runtime_dependency "activesupport", ">= 3.0.0"
-  s.add_runtime_dependency "quandl_operation", "~> 0.2"
 end

data/spec/lib/quandl/data/enumerator_spec.rb

@@ -31,7 +31,7 @@ describe Quandl::Data::Enumerator do
         
         its('to_jd.first.first'){ should be_a Integer }
         its('to_date.first.first'){ should be_a Date }
-    
+        
       end
       
     end

data/spec/lib/quandl/data/format_spec.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Data::Format do
+  
+  expectations = [
+    { 
+      input:  "2013-12-05,10.03,11.0271,13.96623,19.1262\n2013-12-04,10.0760,11.0745,13.9777,19.2117",
+      output: [[ Date.parse('2013-12-05'), 10.03, 11.0271, 13.96623, 19.1262 ], [ Date.parse('2013-12-04'), 10.0760, 11.0745, 13.9777, 19.2117 ]]
+    },
+  ]
+  
+  describe ".data" do
+    expectations.each do |data|
+      it "should recieve:\n#{data[:input].gsub(%Q{\n},'\n')} \n    and return:\n#{data[:output]}" do
+        Quandl::Data::Format.parse(data[:input]).should eq data[:output]
+      end
+    end
+  end
+  
+  subject{  Quandl::Data::Format }
+  let(:escaped_csv){ '2444628,0.00385,0.001,0.123,0.00631,0.534\n2444627,0.00384,0.00159507,0.0056,0.00628948,0.009896' }
+  let(:data_array){ [[ Date.jd(2444628),0.00385,0.001,0.123,0.00631,0.534], [ Date.jd(2444627),0.00384,0.00159507,0.0056,0.00628948,0.009896 ]] }
+
+  let(:csv_data){ "#{Date.today}, 1.0, 2.0" }
+  let(:hash_data){ { Date.today.to_s => [ 1.0, 2.0 ] } }
+  let(:array_data){ [[ Date.today.to_s, 1.0, 2.0 ]] }
+  let(:julian_data){ [[ Date.today.jd, 1.0, 2.0 ]] }
+  let(:date_data){ [[ Date.today, 1.0, 2.0 ]] }
+  
+  it "#hash outputs array" do
+    subject.hash_to_array( hash_data ).should eq array_data
+  end
+  
+  describe "#parse" do
+  
+    [:csv, :hash, :array, :julian, :date].each do |type|
+      context "#{type} data" do
+        it "should eq date_data" do
+          subject.parse( self.send("#{type}_data") ).should eq date_data
+        end
+        it "values should be_a Float" do
+          subject.parse( self.send("#{type}_data") ).each{|r| r[1..-1].each{|v| v.should be_a Float } }
+        end
+      end
+    end
+  
+  
+    let(:csv1){ "2012-12-31,20.0,10.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,27.5,25.0,18.0,33.99,0.0,25.0,0.0,10.0,22.0,34.0,10.0,20.0,26.0,0.0,18.5,25.0,33.0,30.0,20.0,27.5,10.0,19.0,25.0,29.0,23.0,25.0,21.0,30.0,28.0,24.5,33.33,0.0,29.48,10.0,20.0,31.0,0.0,35.0,16.5,19.0,20.0,32.45,25.0,12.5,0.0,25.0,31.4,33.33,38.01,0.0,14.0,20.0,30.0,24.2,15.0,15.0,20.0,12.5,15.0,28.8,12.0,10.0,30.0,25.0,35.0,15.0,30.0,9.0,32.0,34.0,25.0,28.0,30.0,28.0,12.0,35.0,25.0,30.0,10.0,30.0,30.0,19.0,25.0,10.0,16.0,20.0,0.0,27.0,20.0,10.0,17.0,19.0,18.0,34.55,30.0,28.0,0.0,34.5,35.0,26.3,21.17,28.0,17.0,30.0,23.0,25.0,30.0,20.0,30.0,21.0,55.0,24.0,40.0,25.0,0.0,34.0,25.0,20.0,35.0,25.75,29.02,33.0,22.89,20.5,28.3,28.6,22.6,25.25,24.43\n2011-12-31,20.0,10.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,27.5,25.0,24.0,33.99,0.0,0.0,0.0,10.0,22.0,34.0,10.0,20.0,28.0,0.0,20.0,25.0,33.0,30.0,20.0,34.5,10.0,19.0,25.0,29.0,24.0,20.0,21.0,0.0,28.0,26.0,33.33,0.0,29.37,10.0,20.0,31.0,0.0,35.0,16.5,19.0,20.0,32.44,25.0,12.5,0.0,24.0,31.4,33.33,40.69,0.0,14.0,20.0,0.0,22.0,15.0,15.0,20.0,12.5,15.0,28.8,12.0,10.0,0.0,25.0,35.0,15.0,30.0,9.0,32.0,34.0,25.0,28.0,30.0,28.0,12.0,35.0,25.0,30.0,10.0,30.0,30.0,19.0,25.0,10.0,16.0,20.0,0.0,27.0,20.0,10.0,17.0,19.0,20.0,34.55,30.0,28.0,0.0,34.5,35.0,26.3,21.17,28.0,17.0,30.0,30.0,0.0,30.0,20.0,30.0,25.0,55.0,26.0,40.0,25.0,0.0,34.0,25.0,20.0,35.0,25.75,28.55,34.0,23.1,20.88,29.02,28.6,22.8,25.5,24.52\n2010-12-31,20.0,10.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,27.5,25.0,24.0,33.99,0.0,25.0,0.0,10.0,25.0,34.0,10.0,20.0,31.0,0.0,17.0,25.0,33.0,30.0,20.0,0.0,10.0,19.0,25.0,25.0,25.0,20.0,21.0,0.0,28.0,26.0,33.33,0.0,29.41,22.0,24.0,31.0,0.0,25.0,16.5,19.0,18.0,33.99,25.0,12.5,0.0,25.0,31.4,33.33,40.69,0.0,14.0,20.0,0.0,24.2,15.0,15.0,40.0,0.0,15.0,28.59,12.0,10.0,0.0,25.0,35.0,15.0,30.0,9.0,32.0,0.0,25.5,30.0,30.0,28.0,12.0,35.0,27.5,30.0,10.0,30.0,30.0,19.0,25.0,10.0,16.0,20.0,0.0,27.0,20.0,10.0,17.0,19.0,20.0,34.55,30.0,35.0,0.0,0.0,15.0,26.3,21.17,28.0,17.0,30.0,30.0,0.0,30.0,20.0,30.0,25.0,55.0,28.0,40.0,25.0,0.0,34.0,25.0,35.0,35.0,25.75,28.38,35.5,23.96,21.52,27.52,29.0,23.04,25.79,24.71\n2009-12-31,20.0,10.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,27.5,25.0,24.0,33.99,0.0,0.0,0.0,10.0,25.0,34.0,10.0,0.0,33.0,0.0,17.0,25.0,33.0,30.0,20.0,0.0,10.0,20.0,25.0,25.0,25.0,20.0,21.0,0.0,29.0,26.0,33.33,0.0,29.44,27.0,25.0,31.0,0.0,30.0,16.5,16.0,15.0,33.99,28.0,12.5,0.0,26.0,31.4,33.33,40.69,0.0,25.0,20.0,0.0,24.2,15.0,15.0,40.0,0.0,20.0,28.59,12.0,10.0,0.0,25.0,35.0,15.0,28.0,9.0,32.0,0.0,25.5,30.0,30.0,28.0,12.0,35.0,30.0,30.0,10.0,30.0,30.0,19.0,25.0,35.0,16.0,20.0,0.0,27.0,20.0,10.0,18.0,19.0,21.0,34.55,30.0,35.0,0.0,0.0,15.0,26.3,21.17,28.0,25.0,30.0,30.0,0.0,30.0,20.0,30.0,25.0,55.0,28.0,40.0,25.0,0.0,34.0,25.0,35.0,35.0,30.9,28.75,36.5,25.73,21.7,27.96,29.2,23.22,25.73,25.4\n2008-12-31,20.0,10.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,30.0,25.0,24.0,33.99,0.0,0.0,0.0,10.0,25.0,34.0,10.0,0.0,33.5,0.0,17.0,25.0,33.0,30.0,20.0,0.0,10.0,21.0,25.0,25.0,25.0,20.0,21.0,0.0,31.0,26.0,33.33,0.0,29.51,33.0,25.0,31.0,0.0,30.0,16.5,16.0,15.0,33.99,30.0,12.5,0.0,27.0,31.4,33.33,40.69,0.0,25.0,30.0,0.0,27.5,55.0,15.0,40.0,0.0,15.0,29.63,12.0,10.0,0.0,26.0,35.0,15.0,28.0,9.0,32.0,0.0,25.5,30.0,30.0,28.0,12.0,35.0,30.0,30.0,10.0,30.0,35.0,19.0,25.0,35.0,16.0,24.0,0.0,27.0,20.0,10.0,18.0,19.0,22.0,34.55,30.0,35.0,0.0,0.0,15.0,28.0,21.17,28.0,25.0,0.0,30.0,0.0,30.0,20.0,30.0,25.0,55.0,30.0,40.0,25.0,0.0,34.0,28.0,35.0,35.0,30.9,28.65,36.75,27.99,22.0,27.96,29.6,23.29,26.08,26.12\n2007-12-31,20.0,20.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,30.0,25.0,24.0,33.99,0.0,0.0,0.0,10.0,25.0,34.0,10.0,0.0,36.1,0.0,17.0,33.0,34.0,30.0,20.0,0.0,10.0,24.0,25.0,25.0,25.0,20.0,22.0,0.0,31.0,26.0,33.33,0.0,38.36,35.0,25.0,31.0,0.0,30.0,17.5,16.0,18.0,33.99,30.0,12.5,0.0,29.0,37.25,33.33,40.69,0.0,25.0,30.0,0.0,27.5,55.0,15.0,40.0,0.0,15.0,29.63,12.0,12.0,0.0,27.0,35.0,22.5,28.0,9.0,32.0,0.0,25.5,33.0,30.0,28.0,12.0,35.0,30.0,30.0,10.0,30.0,35.0,19.0,25.0,35.0,16.0,24.0,0.0,27.0,20.0,10.0,20.0,19.0,23.0,36.89,32.5,35.0,0.0,0.0,30.0,28.0,21.32,28.0,25.0,0.0,30.0,0.0,30.0,20.0,30.0,25.0,55.0,30.0,40.0,30.0,0.0,34.0,28.0,35.0,35.0,30.9,30.56,38.05,28.46,23.01,28.3,30.2,24.11,27.08,26.96\n"}
+    let(:csv2){ "2012-03-07,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-06,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-05,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-04,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-29,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-28,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n" }
+
+    [:csv1, :csv2].each do |type|
+      context "#{type} data" do
+        it "dates should be_a Date" do
+          subject.parse( self.send(type) ).each{|r| r[0].should be_a Date }
+        end
+        
+        it "values should be_a Float" do
+          subject.parse( self.send(type) ).each do |row|
+            row[1..-1].each do |value|
+              next if value.nil?
+              value.should be_a Float
+            end
+          end
+        end
+        
+      end
+    end
+  
+    let(:invalid1){ "Date, Column 1, Column 2, C3, C4, C5, C6, C7\n 2012-03-07,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-06,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-05,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-04,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-29,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-28,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n" }
+    let(:invalid2){ "2012-03-07,,69.75,69.75,69.75,0.0,0.0,0.0\nDate, Column 1, Column 2, C3, C4, C5, C6, C7\n2012-03-06,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-05,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-04,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-29,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-28,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n" }
+    
+    [:invalid1, :invalid2].each do |type|
+      it "#{type} should raise Quandl::Errors::UnknownDateFormat" do
+        expect {  Quandl::Data::Format.parse( self.send(type) ) }.to raise_error Quandl::Errors::UnknownDateFormat
+      end
+    end
+    
+  end
+  
+end

data/spec/lib/quandl/data/operations_spec.rb

@@ -54,12 +54,12 @@ describe Quandl::Data::Operations do
       subject.transform(:rdiff).first[1].should_not eq value
     end
     it "should rdiff_from" do
-      data = Quandl::Data.new([[2456624,3,5],[2456625,5,4],[2456626,15,20]])
-      data.transform(:rdiff_from).to_jd.should eq [[2456624,4,3],[2456625,2,4],[2456626,0,0]]
+      data = Quandl::Data.new [["2013-11-27", 3.0, 5.0], ["2013-11-28", 5.0, 4.0], ["2013-11-29", 15.0, 20.0]]
+      data.transform(:rdiff_from).to_date_str.should eq [["2013-11-27", 4.0, 3.0], ["2013-11-28", 2.0, 4.0], ["2013-11-29", 0.0, 0.0]]
     end
     it "should cumul" do
-      data = Quandl::Data.new( [[2456624, 10], [2456625, 20], [2456626, 30]] )
-      data.transform(:cumul).to_jd.should eq [[2456624, 10], [2456625, 30], [2456626, 60]]
+      data = Quandl::Data.new [["2013-11-27", 10.0], ["2013-11-28", 20.0], ["2013-11-29", 30.0]]
+      data.transform(:cumul).to_date_str.should eq [["2013-11-27", 10.0], ["2013-11-28", 30.0], ["2013-11-29", 60.0]]
     end
   end
   

data/spec/lib/quandl/data_spec.rb

@@ -10,6 +10,16 @@ describe Quandl::Data do
   let(:data){ Quandl::Data.new(csv2) }
   subject{ data }
   
+  describe "#headers" do
+    subject{ Quandl::Data.new(csv3) }
+    let(:expectation){ ['Date', 'Column 1', 'Column 2', 'C3', 'C4', 'C5', 'C6', 'C7'] }
+    its(:headers){ should eq expectation }
+    
+    its('clone.headers'){ should eq expectation }
+    its('to_jd.headers'){ should eq expectation }
+
+  end
+  
   describe "#clone" do
     subject{ data.clone }
     it{ should be_a Quandl::Data }
@@ -34,10 +44,6 @@ describe Quandl::Data do
       end
       
     end
-    
-    it "csv3 should raise Quandl::Operation::Errors::UnknownDateFormat" do
-      expect { Quandl::Data.new( csv3 ).to_jd }.to raise_error Quandl::Operation::Errors::UnknownDateFormat
-    end
   end
   
   [:to_jd, :sort_ascending, :sort_descending, :to_date_str].each do |o1|

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: quandl_data
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Blake Hilscher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-04 00:00:00.000000000 Z
+date: 2013-12-08 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: quandl_operation
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.1
+- !ruby/object:Gem::Dependency
+  name: quandl_babelfish
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -108,34 +136,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 3.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 3.0.0
-- !ruby/object:Gem::Dependency
-  name: quandl_operation
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.2'
 description: Data will be interfaced
 email:
 - blake@hilscher.ca
@@ -154,15 +154,19 @@ files:
 - Rakefile
 - UPGRADE.md
 - lib/quandl/data.rb
+- lib/quandl/data/babelfisher.rb
 - lib/quandl/data/enumerator.rb
+- lib/quandl/data/format.rb
 - lib/quandl/data/logging.rb
 - lib/quandl/data/operations.rb
 - lib/quandl/data/version.rb
+- lib/quandl/errors/unknown_date_format.rb
 - lib/quandl/fabricate.rb
 - lib/quandl/fabricate/data.rb
 - quandl_data.gemspec
 - spec/lib/fabricate/data_spec.rb
 - spec/lib/quandl/data/enumerator_spec.rb
+- spec/lib/quandl/data/format_spec.rb
 - spec/lib/quandl/data/operations_spec.rb
 - spec/lib/quandl/data_spec.rb
 - spec/spec_helper.rb
@@ -193,6 +197,7 @@ summary: For interfacing with data
 test_files:
 - spec/lib/fabricate/data_spec.rb
 - spec/lib/quandl/data/enumerator_spec.rb
+- spec/lib/quandl/data/format_spec.rb
 - spec/lib/quandl/data/operations_spec.rb
 - spec/lib/quandl/data_spec.rb
 - spec/spec_helper.rb