quandl_operation 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Guardfile +8 -0
- data/README.md +0 -4
- data/UPGRADE.md +5 -0
- data/lib/quandl/operation.rb +3 -3
- data/lib/quandl/operation/collapse.rb +3 -5
- data/lib/quandl/operation/sort.rb +33 -0
- data/lib/quandl/operation/transform.rb +3 -5
- data/lib/quandl/operation/version.rb +1 -1
- data/quandl_operation.gemspec +6 -5
- data/spec/lib/quandl/operation/collapse_spec.rb +2 -3
- metadata +42 -15
- data/lib/quandl/operation/parse.rb +0 -241
- data/spec/lib/quandl/operation/parse_spec.rb +0 -69
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA1:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: 74609f6b618b1b78066b6882c6c71479f41c023d
         | 
| 4 | 
            +
              data.tar.gz: 966ac5e1642141dd2d332032afdba05b3aa21ce9
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 8cfbb02a260e6bd7bd001690216e3ffe15cecb00d3821a2cf7ad5ace8d02c731b87afbb6b1f7f0b37f65a831f5728db6200e2066a46d8fa0a4c0149c793aad78
         | 
| 7 | 
            +
              data.tar.gz: eb57623a02f7ee34fd5aa7229e7df1d327b2a138d4c2fd40f06f83eaa7f53eed2942f56e10bab15f8833541d94eab9029608ed300f59056d59f0169651858d49
         | 
    
        data/Guardfile
    ADDED
    
    
    
        data/README.md
    CHANGED
    
    
    
        data/UPGRADE.md
    CHANGED
    
    
    
        data/lib/quandl/operation.rb
    CHANGED
    
    | @@ -12,11 +12,11 @@ require "active_support/core_ext/object" | |
| 12 12 | 
             
            require 'quandl/operation/core_ext'
         | 
| 13 13 | 
             
            require 'quandl/operation/collapse'
         | 
| 14 14 | 
             
            require 'quandl/operation/transform'
         | 
| 15 | 
            -
            require 'quandl/operation/ | 
| 15 | 
            +
            require 'quandl/operation/sort'
         | 
| 16 16 | 
             
            require 'quandl/operation/errors'
         | 
| 17 17 | 
             
            require 'quandl/operation/qdate'
         | 
| 18 18 |  | 
| 19 19 | 
             
            module Quandl
         | 
| 20 | 
            -
             | 
| 21 | 
            -
             | 
| 20 | 
            +
            module Operation
         | 
| 21 | 
            +
            end
         | 
| 22 22 | 
             
            end
         | 
| @@ -12,16 +12,14 @@ class Collapse | |
| 12 12 | 
             
                  assert_valid_arguments!(data, type)
         | 
| 13 13 | 
             
                  # nothing to do with an empty array
         | 
| 14 14 | 
             
                  return data unless data.compact.present?
         | 
| 15 | 
            -
                  # ensure data is in expected format
         | 
| 16 | 
            -
                  data = Parse.to_jd(data)
         | 
| 17 15 | 
             
                  # source order
         | 
| 18 | 
            -
                  order =  | 
| 16 | 
            +
                  order = Sort.order?(data)
         | 
| 19 17 | 
             
                  # operations expect data in ascending order
         | 
| 20 | 
            -
                  data =  | 
| 18 | 
            +
                  data = Sort.asc(data)
         | 
| 21 19 | 
             
                  # collapse
         | 
| 22 20 | 
             
                  data = collapse(data, type)
         | 
| 23 21 | 
             
                  # return to original order
         | 
| 24 | 
            -
                  data =  | 
| 22 | 
            +
                  data = Sort.desc(data) if order == :desc
         | 
| 25 23 | 
             
                  # onwards
         | 
| 26 24 | 
             
                  data
         | 
| 27 25 | 
             
                end
         | 
| @@ -0,0 +1,33 @@ | |
| 1 | 
            +
            module Quandl
         | 
| 2 | 
            +
            module Operation
         | 
| 3 | 
            +
            class Sort
         | 
| 4 | 
            +
              
         | 
| 5 | 
            +
              class << self
         | 
| 6 | 
            +
                
         | 
| 7 | 
            +
                def order?(data)
         | 
| 8 | 
            +
                  return :none if data.blank? || data[0].blank? || data[1].blank?
         | 
| 9 | 
            +
                  data[0][0] > data[1][0] ? :desc : :asc
         | 
| 10 | 
            +
                end
         | 
| 11 | 
            +
                
         | 
| 12 | 
            +
                def order(data, order = :asc)
         | 
| 13 | 
            +
                  # ascending
         | 
| 14 | 
            +
                  case order
         | 
| 15 | 
            +
                  when :asc   then data = sort_asc(data)
         | 
| 16 | 
            +
                  when :desc  then data = sort_desc(data)
         | 
| 17 | 
            +
                  end
         | 
| 18 | 
            +
                  data
         | 
| 19 | 
            +
                end
         | 
| 20 | 
            +
             | 
| 21 | 
            +
                def asc(data)
         | 
| 22 | 
            +
                  data.sort_by{|r| r[0] }
         | 
| 23 | 
            +
                end
         | 
| 24 | 
            +
                
         | 
| 25 | 
            +
                def desc(data)
         | 
| 26 | 
            +
                  data.sort_by{|r| r[0] }.reverse
         | 
| 27 | 
            +
                end
         | 
| 28 | 
            +
                    
         | 
| 29 | 
            +
              end
         | 
| 30 | 
            +
             | 
| 31 | 
            +
            end
         | 
| 32 | 
            +
            end
         | 
| 33 | 
            +
            end
         | 
| @@ -10,15 +10,13 @@ class Transform | |
| 10 10 | 
             
                  # nothing to do with an empty array
         | 
| 11 11 | 
             
                  return data unless data.compact.present? 
         | 
| 12 12 | 
             
                  # original order
         | 
| 13 | 
            -
                  order =  | 
| 13 | 
            +
                  order = Sort.order?(data)
         | 
| 14 14 | 
             
                  # operations expect data in ascending order
         | 
| 15 | 
            -
                  data =  | 
| 16 | 
            -
                  # operations expect float values
         | 
| 17 | 
            -
                  data = Parse.values_to_float(data)
         | 
| 15 | 
            +
                  data = Sort.asc(data)
         | 
| 18 16 | 
             
                  # transform
         | 
| 19 17 | 
             
                  data = transform( data, type)
         | 
| 20 18 | 
             
                  # return to original order
         | 
| 21 | 
            -
                  data =  | 
| 19 | 
            +
                  data = Sort.desc(data) if order == :desc
         | 
| 22 20 | 
             
                  # onwards
         | 
| 23 21 | 
             
                  data
         | 
| 24 22 | 
             
                end
         | 
    
        data/quandl_operation.gemspec
    CHANGED
    
    | @@ -16,15 +16,16 @@ Gem::Specification.new do |s| | |
| 16 16 | 
             
              s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
         | 
| 17 17 | 
             
              s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
         | 
| 18 18 | 
             
              s.require_paths = ["lib"]
         | 
| 19 | 
            -
             | 
| 19 | 
            +
              
         | 
| 20 | 
            +
              s.add_runtime_dependency "quandl_logger", "~> 0.2"
         | 
| 21 | 
            +
              s.add_runtime_dependency "activesupport", ">= 3.0.0"
         | 
| 22 | 
            +
              
         | 
| 20 23 | 
             
              s.add_development_dependency "rake", "~> 10.0"
         | 
| 21 24 | 
             
              s.add_development_dependency "rspec", "~> 2.13"
         | 
| 22 25 | 
             
              s.add_development_dependency "fivemat", "~> 1.2"
         | 
| 23 26 | 
             
              s.add_development_dependency "pry"
         | 
| 24 27 | 
             
              s.add_development_dependency "simplecov"
         | 
| 25 | 
            -
              
         | 
| 26 28 | 
             
              s.add_development_dependency "quandl_data"
         | 
| 27 | 
            -
              s. | 
| 28 | 
            -
              s. | 
| 29 | 
            -
              
         | 
| 29 | 
            +
              s.add_development_dependency "guard"
         | 
| 30 | 
            +
              s.add_development_dependency "guard-rspec"
         | 
| 30 31 | 
             
            end
         | 
| @@ -46,10 +46,9 @@ describe Quandl::Operation::Collapse do | |
| 46 46 | 
             
              describe ".perform" do
         | 
| 47 47 |  | 
| 48 48 | 
             
                let(:data){
         | 
| 49 | 
            -
                   | 
| 50 | 
            -
                   [2456518, 55.7, nil, nil], [2456506, nil, 136133.0, nil], 
         | 
| 49 | 
            +
                  [[2456537, 56.2, nil, nil], [2456518, 55.7, nil, nil], [2456506, nil, 136133.0, nil], 
         | 
| 51 50 | 
             
                   [2456487, 55.4, nil, nil], [2456475, nil, 135964.0, nil], 
         | 
| 52 | 
            -
                   [2456457, 50.9, nil, nil], [2456445, nil, 135860.0, nil]] | 
| 51 | 
            +
                   [2456457, 50.9, nil, nil], [2456445, nil, 135860.0, nil]]
         | 
| 53 52 | 
             
                }
         | 
| 54 53 |  | 
| 55 54 | 
             
                it "should collapse data with nils" do
         | 
    
        metadata
    CHANGED
    
    | @@ -1,15 +1,43 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: quandl_operation
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0. | 
| 4 | 
            +
              version: 0.3.0
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Blake Hilscher
         | 
| 8 8 | 
             
            autorequire: 
         | 
| 9 9 | 
             
            bindir: bin
         | 
| 10 10 | 
             
            cert_chain: []
         | 
| 11 | 
            -
            date: 2013- | 
| 11 | 
            +
            date: 2013-12-06 00:00:00.000000000 Z
         | 
| 12 12 | 
             
            dependencies:
         | 
| 13 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 14 | 
            +
              name: quandl_logger
         | 
| 15 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 16 | 
            +
                requirements:
         | 
| 17 | 
            +
                - - ~>
         | 
| 18 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 19 | 
            +
                    version: '0.2'
         | 
| 20 | 
            +
              type: :runtime
         | 
| 21 | 
            +
              prerelease: false
         | 
| 22 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 23 | 
            +
                requirements:
         | 
| 24 | 
            +
                - - ~>
         | 
| 25 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 26 | 
            +
                    version: '0.2'
         | 
| 27 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 28 | 
            +
              name: activesupport
         | 
| 29 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 30 | 
            +
                requirements:
         | 
| 31 | 
            +
                - - '>='
         | 
| 32 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 33 | 
            +
                    version: 3.0.0
         | 
| 34 | 
            +
              type: :runtime
         | 
| 35 | 
            +
              prerelease: false
         | 
| 36 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 37 | 
            +
                requirements:
         | 
| 38 | 
            +
                - - '>='
         | 
| 39 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 40 | 
            +
                    version: 3.0.0
         | 
| 13 41 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 14 42 | 
             
              name: rake
         | 
| 15 43 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| @@ -95,33 +123,33 @@ dependencies: | |
| 95 123 | 
             
                  - !ruby/object:Gem::Version
         | 
| 96 124 | 
             
                    version: '0'
         | 
| 97 125 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 98 | 
            -
              name:  | 
| 126 | 
            +
              name: guard
         | 
| 99 127 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| 100 128 | 
             
                requirements:
         | 
| 101 129 | 
             
                - - '>='
         | 
| 102 130 | 
             
                  - !ruby/object:Gem::Version
         | 
| 103 | 
            -
                    version:  | 
| 104 | 
            -
              type: : | 
| 131 | 
            +
                    version: '0'
         | 
| 132 | 
            +
              type: :development
         | 
| 105 133 | 
             
              prerelease: false
         | 
| 106 134 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 107 135 | 
             
                requirements:
         | 
| 108 136 | 
             
                - - '>='
         | 
| 109 137 | 
             
                  - !ruby/object:Gem::Version
         | 
| 110 | 
            -
                    version:  | 
| 138 | 
            +
                    version: '0'
         | 
| 111 139 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 112 | 
            -
              name:  | 
| 140 | 
            +
              name: guard-rspec
         | 
| 113 141 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| 114 142 | 
             
                requirements:
         | 
| 115 | 
            -
                - -  | 
| 143 | 
            +
                - - '>='
         | 
| 116 144 | 
             
                  - !ruby/object:Gem::Version
         | 
| 117 | 
            -
                    version: '0 | 
| 118 | 
            -
              type: : | 
| 145 | 
            +
                    version: '0'
         | 
| 146 | 
            +
              type: :development
         | 
| 119 147 | 
             
              prerelease: false
         | 
| 120 148 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 121 149 | 
             
                requirements:
         | 
| 122 | 
            -
                - -  | 
| 150 | 
            +
                - - '>='
         | 
| 123 151 | 
             
                  - !ruby/object:Gem::Version
         | 
| 124 | 
            -
                    version: '0 | 
| 152 | 
            +
                    version: '0'
         | 
| 125 153 | 
             
            description: Data will be operated
         | 
| 126 154 | 
             
            email:
         | 
| 127 155 | 
             
            - blake@hilscher.ca
         | 
| @@ -134,6 +162,7 @@ files: | |
| 134 162 | 
             
            - .travis.yml
         | 
| 135 163 | 
             
            - .yardopts
         | 
| 136 164 | 
             
            - Gemfile
         | 
| 165 | 
            +
            - Guardfile
         | 
| 137 166 | 
             
            - LICENSE
         | 
| 138 167 | 
             
            - README.md
         | 
| 139 168 | 
             
            - Rakefile
         | 
| @@ -148,14 +177,13 @@ files: | |
| 148 177 | 
             
            - lib/quandl/operation/core_ext/string.rb
         | 
| 149 178 | 
             
            - lib/quandl/operation/core_ext/time.rb
         | 
| 150 179 | 
             
            - lib/quandl/operation/errors.rb
         | 
| 151 | 
            -
            - lib/quandl/operation/parse.rb
         | 
| 152 180 | 
             
            - lib/quandl/operation/qdate.rb
         | 
| 181 | 
            +
            - lib/quandl/operation/sort.rb
         | 
| 153 182 | 
             
            - lib/quandl/operation/transform.rb
         | 
| 154 183 | 
             
            - lib/quandl/operation/version.rb
         | 
| 155 184 | 
             
            - quandl_operation.gemspec
         | 
| 156 185 | 
             
            - spec/lib/quandl/operation/collapse_spec.rb
         | 
| 157 186 | 
             
            - spec/lib/quandl/operation/date_spec.rb
         | 
| 158 | 
            -
            - spec/lib/quandl/operation/parse_spec.rb
         | 
| 159 187 | 
             
            - spec/lib/quandl/operation/transform_spec.rb
         | 
| 160 188 | 
             
            - spec/spec_helper.rb
         | 
| 161 189 | 
             
            homepage: http://blake.hilscher.ca/
         | 
| @@ -185,6 +213,5 @@ summary: For altering data | |
| 185 213 | 
             
            test_files:
         | 
| 186 214 | 
             
            - spec/lib/quandl/operation/collapse_spec.rb
         | 
| 187 215 | 
             
            - spec/lib/quandl/operation/date_spec.rb
         | 
| 188 | 
            -
            - spec/lib/quandl/operation/parse_spec.rb
         | 
| 189 216 | 
             
            - spec/lib/quandl/operation/transform_spec.rb
         | 
| 190 217 | 
             
            - spec/spec_helper.rb
         | 
| @@ -1,241 +0,0 @@ | |
| 1 | 
            -
            module Quandl
         | 
| 2 | 
            -
            module Operation
         | 
| 3 | 
            -
             | 
| 4 | 
            -
            class Parse
         | 
| 5 | 
            -
              
         | 
| 6 | 
            -
              class << self
         | 
| 7 | 
            -
              
         | 
| 8 | 
            -
                def perform(data)
         | 
| 9 | 
            -
                  return [] if data.blank?
         | 
| 10 | 
            -
                  data = hash_to_array(data)
         | 
| 11 | 
            -
                  data = csv_to_array(data)
         | 
| 12 | 
            -
                  # data = to_jd(data)
         | 
| 13 | 
            -
                  data = to_date(data)
         | 
| 14 | 
            -
                  data.dup
         | 
| 15 | 
            -
                end
         | 
| 16 | 
            -
                
         | 
| 17 | 
            -
                def hash_to_array(data)
         | 
| 18 | 
            -
                  data = data.collect{|k,v| [k] + v } if data.kind_of?(Hash)
         | 
| 19 | 
            -
                  data
         | 
| 20 | 
            -
                end
         | 
| 21 | 
            -
              
         | 
| 22 | 
            -
                def csv_to_array(data)
         | 
| 23 | 
            -
                  if data.is_a?(String)
         | 
| 24 | 
            -
                    data = data.gsub('\n', "\n")
         | 
| 25 | 
            -
                    data = CSV.parse( data )
         | 
| 26 | 
            -
                  end
         | 
| 27 | 
            -
                  data = values_to_float(data)
         | 
| 28 | 
            -
                  data
         | 
| 29 | 
            -
                end
         | 
| 30 | 
            -
                
         | 
| 31 | 
            -
                def sort(data, order = :asc)
         | 
| 32 | 
            -
                  # ascending
         | 
| 33 | 
            -
                  case order
         | 
| 34 | 
            -
                  when :asc   then data = sort_asc(data)
         | 
| 35 | 
            -
                  when :desc  then data = sort_desc(data)
         | 
| 36 | 
            -
                  end
         | 
| 37 | 
            -
                  data
         | 
| 38 | 
            -
                end
         | 
| 39 | 
            -
              
         | 
| 40 | 
            -
                def sort_order?(data)
         | 
| 41 | 
            -
                  return :none if data.blank? || data[0].blank? || data[1].blank?
         | 
| 42 | 
            -
                  data[0][0] > data[1][0] ? :desc : :asc
         | 
| 43 | 
            -
                end
         | 
| 44 | 
            -
              
         | 
| 45 | 
            -
                def sort_asc(data)
         | 
| 46 | 
            -
                  data.sort_by{|r| r[0] }
         | 
| 47 | 
            -
                end
         | 
| 48 | 
            -
              
         | 
| 49 | 
            -
                def sort_desc(data)
         | 
| 50 | 
            -
                  data.sort_by{|r| r[0] }.reverse
         | 
| 51 | 
            -
                end
         | 
| 52 | 
            -
                
         | 
| 53 | 
            -
                def to_date(data)
         | 
| 54 | 
            -
                  return data if data_missing_rows?(data)
         | 
| 55 | 
            -
                  # guess the current date format
         | 
| 56 | 
            -
                  format = date_format?(data)
         | 
| 57 | 
            -
                  # convert dates to Date
         | 
| 58 | 
            -
                  case format
         | 
| 59 | 
            -
                  when :date            then return data
         | 
| 60 | 
            -
                  when :date_string     then return date_strings_to_date( data )
         | 
| 61 | 
            -
                  when :jd, :jd_string  then return jds_to_date( data )
         | 
| 62 | 
            -
                  when :unknown         then raise_date_format_error!( data[0] )
         | 
| 63 | 
            -
                  end
         | 
| 64 | 
            -
                  # return data
         | 
| 65 | 
            -
                  data
         | 
| 66 | 
            -
                end
         | 
| 67 | 
            -
                
         | 
| 68 | 
            -
                def to_jd(data)
         | 
| 69 | 
            -
                  return data if data_missing_rows?(data)
         | 
| 70 | 
            -
                  # guess the current date format
         | 
| 71 | 
            -
                  format = date_format?(data)
         | 
| 72 | 
            -
                  # convert dates to Date
         | 
| 73 | 
            -
                  case format
         | 
| 74 | 
            -
                  when :jd          then return data
         | 
| 75 | 
            -
                  when :jd_string   then return jd_strings_to_jd( data )
         | 
| 76 | 
            -
                  when :date        then return dates_to_jd( data )
         | 
| 77 | 
            -
                  when :date_string then return date_strings_to_jd( data )
         | 
| 78 | 
            -
                  when :unknown     then raise_date_format_error!( data[0] )
         | 
| 79 | 
            -
                  end
         | 
| 80 | 
            -
                  # return data
         | 
| 81 | 
            -
                  data
         | 
| 82 | 
            -
                end
         | 
| 83 | 
            -
                
         | 
| 84 | 
            -
                def date_format?(data)
         | 
| 85 | 
            -
                  value = data[0][0]
         | 
| 86 | 
            -
                  # julian date?
         | 
| 87 | 
            -
                  return :date          if value.is_a?(Date)
         | 
| 88 | 
            -
                  return :jd            if value.is_a?(Integer)
         | 
| 89 | 
            -
                  return :date_string   if value.is_a?(String) && value =~ /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/
         | 
| 90 | 
            -
                  return :jd_string     if value.kind_of?(String) && value.numeric?
         | 
| 91 | 
            -
                  return :unknown
         | 
| 92 | 
            -
                end
         | 
| 93 | 
            -
                
         | 
| 94 | 
            -
                def dates_to_jd(data)
         | 
| 95 | 
            -
                  return data if data_missing_rows?(data) || data[0][0].is_a?(Integer)
         | 
| 96 | 
            -
                  # dont alter by reference
         | 
| 97 | 
            -
                  result = []
         | 
| 98 | 
            -
                  # for each row
         | 
| 99 | 
            -
                  data.each_with_index do |row, index|
         | 
| 100 | 
            -
                    # copy
         | 
| 101 | 
            -
                    row = row.dup
         | 
| 102 | 
            -
                    # string to date
         | 
| 103 | 
            -
                    date = row[0]
         | 
| 104 | 
            -
                    date = Date.parse(row[0]) if row[0].is_a?(String)
         | 
| 105 | 
            -
                    # date to julian
         | 
| 106 | 
            -
                    row[0] = date.jd if date.respond_to?(:jd)
         | 
| 107 | 
            -
                    # save result
         | 
| 108 | 
            -
                    result[index] = row
         | 
| 109 | 
            -
                  end
         | 
| 110 | 
            -
                  # all done
         | 
| 111 | 
            -
                  result
         | 
| 112 | 
            -
                end
         | 
| 113 | 
            -
                
         | 
| 114 | 
            -
                
         | 
| 115 | 
            -
                ###################
         | 
| 116 | 
            -
                # DATES TO JULIAN #
         | 
| 117 | 
            -
                ###################
         | 
| 118 | 
            -
             | 
| 119 | 
            -
                def jd_strings_to_jd(data)
         | 
| 120 | 
            -
                  # skip when already formatted correctly
         | 
| 121 | 
            -
                  return data if data_missing_rows?(data) || data[0][0].is_a?(Integer)
         | 
| 122 | 
            -
                  # otherwise cast string jds to int
         | 
| 123 | 
            -
                  output = []
         | 
| 124 | 
            -
                  data.each_with_index do |row, index|
         | 
| 125 | 
            -
                    output << parse_jd_string(row) rescue raise_date_format_error!( row, index, :jd_strings_to_jd )
         | 
| 126 | 
            -
                  end
         | 
| 127 | 
            -
                  output
         | 
| 128 | 
            -
                end
         | 
| 129 | 
            -
                
         | 
| 130 | 
            -
                def date_strings_to_jd(data)
         | 
| 131 | 
            -
                  # skip when already formatted correctly
         | 
| 132 | 
            -
                  return data if data_missing_rows?(data) || data[0][0].is_a?(Date)
         | 
| 133 | 
            -
                  # otherwise cast string jds to int
         | 
| 134 | 
            -
                  output = []
         | 
| 135 | 
            -
                  data.each_with_index do |row, index|
         | 
| 136 | 
            -
                    row = parse_date_string(row) rescue raise_date_format_error!( row, index, :date_strings_to_jd )
         | 
| 137 | 
            -
                    row[0] = row[0].jd
         | 
| 138 | 
            -
                    output << row
         | 
| 139 | 
            -
                  end
         | 
| 140 | 
            -
                  output
         | 
| 141 | 
            -
                end
         | 
| 142 | 
            -
             | 
| 143 | 
            -
                
         | 
| 144 | 
            -
                #################
         | 
| 145 | 
            -
                # DATES TO DATE #
         | 
| 146 | 
            -
                #################
         | 
| 147 | 
            -
             | 
| 148 | 
            -
                def jds_to_date(data)
         | 
| 149 | 
            -
                  return data if data_missing_rows?(data) || data[0][0].is_a?(Date)
         | 
| 150 | 
            -
                  output = []
         | 
| 151 | 
            -
                  data.each_with_index do |row, index|
         | 
| 152 | 
            -
                    output << parse_jd(row) rescue raise_date_format_error!( row, index, :jds_to_date )
         | 
| 153 | 
            -
                  end
         | 
| 154 | 
            -
                  output
         | 
| 155 | 
            -
                end
         | 
| 156 | 
            -
              
         | 
| 157 | 
            -
                def date_strings_to_date(data)
         | 
| 158 | 
            -
                  # skip when already formatted correctly
         | 
| 159 | 
            -
                  return data if data_missing_rows?(data) || data[0][0].is_a?(Date)
         | 
| 160 | 
            -
                  # otherwise cast string jds to int
         | 
| 161 | 
            -
                  output = []
         | 
| 162 | 
            -
                  data.each_with_index do |row, index|
         | 
| 163 | 
            -
                    output << parse_date_string(row) rescue raise_date_format_error!( row, index, :date_strings_to_date )
         | 
| 164 | 
            -
                  end
         | 
| 165 | 
            -
                  output
         | 
| 166 | 
            -
                end
         | 
| 167 | 
            -
                
         | 
| 168 | 
            -
                
         | 
| 169 | 
            -
                ###################
         | 
| 170 | 
            -
                # VALUES TO FLOAT #
         | 
| 171 | 
            -
                ###################
         | 
| 172 | 
            -
             | 
| 173 | 
            -
                def values_to_float(data)
         | 
| 174 | 
            -
                  # skip unless value is a string
         | 
| 175 | 
            -
                  return data if data_missing_rows?(data) || data[0][1].is_a?(Float)
         | 
| 176 | 
            -
                  # cast values to float
         | 
| 177 | 
            -
                  data.collect do |row|
         | 
| 178 | 
            -
                    new_row = [row[0]]
         | 
| 179 | 
            -
                    row[1..-1].each_with_index do |value, index|
         | 
| 180 | 
            -
                      value = value.to_f if value.is_a?(String)
         | 
| 181 | 
            -
                      new_row[index + 1] = value
         | 
| 182 | 
            -
                    end
         | 
| 183 | 
            -
                    new_row
         | 
| 184 | 
            -
                  end
         | 
| 185 | 
            -
                end
         | 
| 186 | 
            -
                
         | 
| 187 | 
            -
                
         | 
| 188 | 
            -
                def data_missing_rows?(data)
         | 
| 189 | 
            -
                  !data_has_rows?(data)
         | 
| 190 | 
            -
                end
         | 
| 191 | 
            -
                
         | 
| 192 | 
            -
                def data_has_rows?(data)
         | 
| 193 | 
            -
                  data.is_a?(Array) && data[0].is_a?(Array) && data[0][0].present?
         | 
| 194 | 
            -
                end
         | 
| 195 | 
            -
                
         | 
| 196 | 
            -
                
         | 
| 197 | 
            -
                protected
         | 
| 198 | 
            -
                
         | 
| 199 | 
            -
                def parse_jd(row)
         | 
| 200 | 
            -
                  # parse jd_string
         | 
| 201 | 
            -
                  row = parse_jd_string(row)
         | 
| 202 | 
            -
                  # jd to date
         | 
| 203 | 
            -
                  row[0] = Date.jd( row[0] )
         | 
| 204 | 
            -
                  # onwards
         | 
| 205 | 
            -
                  row
         | 
| 206 | 
            -
                end
         | 
| 207 | 
            -
                
         | 
| 208 | 
            -
                def parse_jd_string(row)
         | 
| 209 | 
            -
                  row = row.dup
         | 
| 210 | 
            -
                  row[0] = row[0].to_i
         | 
| 211 | 
            -
                  # dont allow dates that are before 0000
         | 
| 212 | 
            -
                  raise Errors::UnknownDateFormat if row[0] <= 1721058
         | 
| 213 | 
            -
                  row
         | 
| 214 | 
            -
                end
         | 
| 215 | 
            -
                
         | 
| 216 | 
            -
                def parse_date_string(row)
         | 
| 217 | 
            -
                  row = row.dup
         | 
| 218 | 
            -
                  # extract date
         | 
| 219 | 
            -
                  date = row[0]
         | 
| 220 | 
            -
                  # split date into parts
         | 
| 221 | 
            -
                  date_values = date.split('-').collect(&:to_i)
         | 
| 222 | 
            -
                  # ensure date is valid
         | 
| 223 | 
            -
                  raise Errors::UnknownDateFormat unless date_values.count == 3
         | 
| 224 | 
            -
                  # add to row
         | 
| 225 | 
            -
                  row[0] = Date.new( *date_values )
         | 
| 226 | 
            -
                  row
         | 
| 227 | 
            -
                end
         | 
| 228 | 
            -
                
         | 
| 229 | 
            -
                
         | 
| 230 | 
            -
                private
         | 
| 231 | 
            -
                
         | 
| 232 | 
            -
                def raise_date_format_error!(row, index = 0, type = :none)
         | 
| 233 | 
            -
                  message = "UnknownDateFormat: '#{row[0]}', index: data[#{index}][0], strategy: '#{type}', row: #{row}"
         | 
| 234 | 
            -
                  raise Errors::UnknownDateFormat, message
         | 
| 235 | 
            -
                end
         | 
| 236 | 
            -
                
         | 
| 237 | 
            -
              end
         | 
| 238 | 
            -
              
         | 
| 239 | 
            -
            end
         | 
| 240 | 
            -
            end
         | 
| 241 | 
            -
            end
         | 
| @@ -1,69 +0,0 @@ | |
| 1 | 
            -
            # encoding: utf-8
         | 
| 2 | 
            -
            require 'spec_helper'
         | 
| 3 | 
            -
             | 
| 4 | 
            -
            describe Quandl::Operation::Parse do
         | 
| 5 | 
            -
              subject{ Quandl::Operation::Parse }
         | 
| 6 | 
            -
              let(:escaped_csv){ '2444628,0.00385,0.001,0.123,0.00631,0.534\n2444627,0.00384,0.00159507,0.0056,0.00628948,0.009896' }
         | 
| 7 | 
            -
              let(:data_array){ [[ Date.jd(2444628),0.00385,0.001,0.123,0.00631,0.534], [ Date.jd(2444627),0.00384,0.00159507,0.0056,0.00628948,0.009896 ]] }
         | 
| 8 | 
            -
             | 
| 9 | 
            -
              let(:csv_data){ "#{Date.today}, 1.0, 2.0" }
         | 
| 10 | 
            -
              let(:hash_data){ { Date.today.to_s => [ 1.0, 2.0 ] } }
         | 
| 11 | 
            -
              let(:array_data){ [[ Date.today.to_s, 1.0, 2.0 ]] }
         | 
| 12 | 
            -
              let(:julian_data){ [[ Date.today.jd, 1.0, 2.0 ]] }
         | 
| 13 | 
            -
              let(:date_data){ [[ Date.today, 1.0, 2.0 ]] }
         | 
| 14 | 
            -
              
         | 
| 15 | 
            -
              it "#hash outputs array" do
         | 
| 16 | 
            -
                subject.hash_to_array( hash_data ).should eq array_data
         | 
| 17 | 
            -
              end
         | 
| 18 | 
            -
              
         | 
| 19 | 
            -
              it "#csv outputs array" do
         | 
| 20 | 
            -
                subject.csv_to_array( csv_data ).should eq array_data
         | 
| 21 | 
            -
              end
         | 
| 22 | 
            -
              
         | 
| 23 | 
            -
              describe "#perform" do
         | 
| 24 | 
            -
              
         | 
| 25 | 
            -
                [:csv, :hash, :array, :julian, :date].each do |type|
         | 
| 26 | 
            -
                  context "#{type} data" do
         | 
| 27 | 
            -
                    it "should eq date_data" do
         | 
| 28 | 
            -
                      subject.perform( self.send("#{type}_data") ).should eq date_data
         | 
| 29 | 
            -
                    end
         | 
| 30 | 
            -
                    it "values should be_a Float" do
         | 
| 31 | 
            -
                      subject.perform( self.send("#{type}_data") ).each{|r| r[1..-1].each{|v| v.should be_a Float } }
         | 
| 32 | 
            -
                    end
         | 
| 33 | 
            -
                  end
         | 
| 34 | 
            -
                end
         | 
| 35 | 
            -
              
         | 
| 36 | 
            -
              
         | 
| 37 | 
            -
                let(:csv1){ "2012-12-31,20.0,10.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,27.5,25.0,18.0,33.99,0.0,25.0,0.0,10.0,22.0,34.0,10.0,20.0,26.0,0.0,18.5,25.0,33.0,30.0,20.0,27.5,10.0,19.0,25.0,29.0,23.0,25.0,21.0,30.0,28.0,24.5,33.33,0.0,29.48,10.0,20.0,31.0,0.0,35.0,16.5,19.0,20.0,32.45,25.0,12.5,0.0,25.0,31.4,33.33,38.01,0.0,14.0,20.0,30.0,24.2,15.0,15.0,20.0,12.5,15.0,28.8,12.0,10.0,30.0,25.0,35.0,15.0,30.0,9.0,32.0,34.0,25.0,28.0,30.0,28.0,12.0,35.0,25.0,30.0,10.0,30.0,30.0,19.0,25.0,10.0,16.0,20.0,0.0,27.0,20.0,10.0,17.0,19.0,18.0,34.55,30.0,28.0,0.0,34.5,35.0,26.3,21.17,28.0,17.0,30.0,23.0,25.0,30.0,20.0,30.0,21.0,55.0,24.0,40.0,25.0,0.0,34.0,25.0,20.0,35.0,25.75,29.02,33.0,22.89,20.5,28.3,28.6,22.6,25.25,24.43\n2011-12-31,20.0,10.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,27.5,25.0,24.0,33.99,0.0,0.0,0.0,10.0,22.0,34.0,10.0,20.0,28.0,0.0,20.0,25.0,33.0,30.0,20.0,34.5,10.0,19.0,25.0,29.0,24.0,20.0,21.0,0.0,28.0,26.0,33.33,0.0,29.37,10.0,20.0,31.0,0.0,35.0,16.5,19.0,20.0,32.44,25.0,12.5,0.0,24.0,31.4,33.33,40.69,0.0,14.0,20.0,0.0,22.0,15.0,15.0,20.0,12.5,15.0,28.8,12.0,10.0,0.0,25.0,35.0,15.0,30.0,9.0,32.0,34.0,25.0,28.0,30.0,28.0,12.0,35.0,25.0,30.0,10.0,30.0,30.0,19.0,25.0,10.0,16.0,20.0,0.0,27.0,20.0,10.0,17.0,19.0,20.0,34.55,30.0,28.0,0.0,34.5,35.0,26.3,21.17,28.0,17.0,30.0,30.0,0.0,30.0,20.0,30.0,25.0,55.0,26.0,40.0,25.0,0.0,34.0,25.0,20.0,35.0,25.75,28.55,34.0,23.1,20.88,29.02,28.6,22.8,25.5,24.52\n2010-12-31,20.0,10.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,27.5,25.0,24.0,33.99,0.0,25.0,0.0,10.0,25.0,34.0,10.0,20.0,31.0,0.0,17.0,25.0,33.0,30.0,20.0,0.0,10.0,19.0,25.0,25.0,25.0,20.0,21.0,0.0,28.0,26.0,33.33,0.0,29.41,22.0,24.0,31.0,0.0,25.0,16.5,19.0,18.0,33.99,25.0,12.5,0.0,25.0,31.4,33.33,40.69,0.0,14.0,20.0,0.0,24.2,15.0,15.0,40.0,0.0,15.0,28.59,12.0,10.0,0.0,25.0,35.0,15.0,30.0,9.0,32.0,0.0,25.5,30.0,30.0,28.0,12.0,35.0,27.5,30.0,10.0,30.0,30.0,19.0,25.0,10.0,16.0,20.0,0.0,27.0,20.0,10.0,17.0,19.0,20.0,34.55,30.0,35.0,0.0,0.0,15.0,26.3,21.17,28.0,17.0,30.0,30.0,0.0,30.0,20.0,30.0,25.0,55.0,28.0,40.0,25.0,0.0,34.0,25.0,35.0,35.0,25.75,28.38,35.5,23.96,21.52,27.52,29.0,23.04,25.79,24.71\n2009-12-31,20.0,10.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,27.5,25.0,24.0,33.99,0.0,0.0,0.0,10.0,25.0,34.0,10.0,0.0,33.0,0.0,17.0,25.0,33.0,30.0,20.0,0.0,10.0,20.0,25.0,25.0,25.0,20.0,21.0,0.0,29.0,26.0,33.33,0.0,29.44,27.0,25.0,31.0,0.0,30.0,16.5,16.0,15.0,33.99,28.0,12.5,0.0,26.0,31.4,33.33,40.69,0.0,25.0,20.0,0.0,24.2,15.0,15.0,40.0,0.0,20.0,28.59,12.0,10.0,0.0,25.0,35.0,15.0,28.0,9.0,32.0,0.0,25.5,30.0,30.0,28.0,12.0,35.0,30.0,30.0,10.0,30.0,30.0,19.0,25.0,35.0,16.0,20.0,0.0,27.0,20.0,10.0,18.0,19.0,21.0,34.55,30.0,35.0,0.0,0.0,15.0,26.3,21.17,28.0,25.0,30.0,30.0,0.0,30.0,20.0,30.0,25.0,55.0,28.0,40.0,25.0,0.0,34.0,25.0,35.0,35.0,30.9,28.75,36.5,25.73,21.7,27.96,29.2,23.22,25.73,25.4\n2008-12-31,20.0,10.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,30.0,25.0,24.0,33.99,0.0,0.0,0.0,10.0,25.0,34.0,10.0,0.0,33.5,0.0,17.0,25.0,33.0,30.0,20.0,0.0,10.0,21.0,25.0,25.0,25.0,20.0,21.0,0.0,31.0,26.0,33.33,0.0,29.51,33.0,25.0,31.0,0.0,30.0,16.5,16.0,15.0,33.99,30.0,12.5,0.0,27.0,31.4,33.33,40.69,0.0,25.0,30.0,0.0,27.5,55.0,15.0,40.0,0.0,15.0,29.63,12.0,10.0,0.0,26.0,35.0,15.0,28.0,9.0,32.0,0.0,25.5,30.0,30.0,28.0,12.0,35.0,30.0,30.0,10.0,30.0,35.0,19.0,25.0,35.0,16.0,24.0,0.0,27.0,20.0,10.0,18.0,19.0,22.0,34.55,30.0,35.0,0.0,0.0,15.0,28.0,21.17,28.0,25.0,0.0,30.0,0.0,30.0,20.0,30.0,25.0,55.0,30.0,40.0,25.0,0.0,34.0,28.0,35.0,35.0,30.9,28.65,36.75,27.99,22.0,27.96,29.6,23.29,26.08,26.12\n2007-12-31,20.0,20.0,35.0,35.0,20.0,28.0,30.0,25.0,0.0,0.0,30.0,25.0,24.0,33.99,0.0,0.0,0.0,10.0,25.0,34.0,10.0,0.0,36.1,0.0,17.0,33.0,34.0,30.0,20.0,0.0,10.0,24.0,25.0,25.0,25.0,20.0,22.0,0.0,31.0,26.0,33.33,0.0,38.36,35.0,25.0,31.0,0.0,30.0,17.5,16.0,18.0,33.99,30.0,12.5,0.0,29.0,37.25,33.33,40.69,0.0,25.0,30.0,0.0,27.5,55.0,15.0,40.0,0.0,15.0,29.63,12.0,12.0,0.0,27.0,35.0,22.5,28.0,9.0,32.0,0.0,25.5,33.0,30.0,28.0,12.0,35.0,30.0,30.0,10.0,30.0,35.0,19.0,25.0,35.0,16.0,24.0,0.0,27.0,20.0,10.0,20.0,19.0,23.0,36.89,32.5,35.0,0.0,0.0,30.0,28.0,21.32,28.0,25.0,0.0,30.0,0.0,30.0,20.0,30.0,25.0,55.0,30.0,40.0,30.0,0.0,34.0,28.0,35.0,35.0,30.9,30.56,38.05,28.46,23.01,28.3,30.2,24.11,27.08,26.96\n"}
         | 
| 38 | 
            -
                let(:csv2){ "2012-03-07,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-06,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-05,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-04,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-29,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-28,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n" }
         | 
| 39 | 
            -
             | 
| 40 | 
            -
                [:csv1, :csv2].each do |type|
         | 
| 41 | 
            -
                  context "#{type} data" do
         | 
| 42 | 
            -
                    it "dates should be_a Date" do
         | 
| 43 | 
            -
                      subject.perform( self.send(type) ).each{|r| r[0].should be_a Date }
         | 
| 44 | 
            -
                    end
         | 
| 45 | 
            -
                    
         | 
| 46 | 
            -
                    it "values should be_a Float" do
         | 
| 47 | 
            -
                      subject.perform( self.send(type) ).each do |row|
         | 
| 48 | 
            -
                        row[1..-1].each do |value|
         | 
| 49 | 
            -
                          next if value.nil?
         | 
| 50 | 
            -
                          value.should be_a Float
         | 
| 51 | 
            -
                        end
         | 
| 52 | 
            -
                      end
         | 
| 53 | 
            -
                    end
         | 
| 54 | 
            -
                    
         | 
| 55 | 
            -
                  end
         | 
| 56 | 
            -
                end
         | 
| 57 | 
            -
              
         | 
| 58 | 
            -
                let(:invalid1){ "Date, Column 1, Column 2, C3, C4, C5, C6, C7\n 2012-03-07,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-06,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-05,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-04,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-29,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-28,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n" }
         | 
| 59 | 
            -
                let(:invalid2){ "2012-03-07,,69.75,69.75,69.75,0.0,0.0,0.0\nDate, Column 1, Column 2, C3, C4, C5, C6, C7\n2012-03-06,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-05,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-03-04,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-29,,69.75,69.75,69.75,0.0,0.0,0.0\n2012-02-28,69.75,69.75,69.75,69.75,0.0,0.0,0.0\n" }
         | 
| 60 | 
            -
                
         | 
| 61 | 
            -
                [:invalid1, :invalid2].each do |type|
         | 
| 62 | 
            -
                  it "#{type} should raise Quandl::Operation::Errors::UnknownDateFormat" do
         | 
| 63 | 
            -
                    expect { Quandl::Operation::Parse.perform( self.send(type) ) }.to raise_error Quandl::Operation::Errors::UnknownDateFormat
         | 
| 64 | 
            -
                  end
         | 
| 65 | 
            -
                end
         | 
| 66 | 
            -
                
         | 
| 67 | 
            -
              end
         | 
| 68 | 
            -
              
         | 
| 69 | 
            -
            end
         |