quandl_operation 0.1.11 → 0.1.12

data/UPGRADE.md

@@ -1,3 +1,8 @@
+## 0.1.12
+
+* Quandl::Operation::Collapse.perform will merge rows with nils
+
+
 ## 0.1.11
 
 * it should parse escaped csv data

data/lib/quandl/operation/collapse.rb

@@ -9,8 +9,15 @@ class Collapse
   class << self
   
     def perform(data, frequency)
-      data = Parse.sort( data )
+      # source order
+      order = Parse.sort_order?(data)
+      # operations expect data in ascending order
+      data = Parse.sort( data, :asc )
+      # collapse
       data = collapse_and_log(data, frequency)
+      # return to original order
+      data = Parse.sort( data, :desc ) if order == :desc
+      # onwards
       data
     end
     
@@ -40,13 +47,18 @@ class Collapse
       range = find_end_of_range( data[0][0], frequency )
       # iterate over the data
       data.each do |row|
-        # grab date and value
+        # grab date and values
         date, value = row[0], row[1..-1]
         value = value.first if value.count == 1
         # bump to the next range if it exceeds the current one
         range = find_end_of_range(date, frequency) unless inside_range?(date, range)
         # consider the value for the next range
-        collapsed_data[range] = value if inside_range?(date, range) && value.present?
+        if inside_range?(date, range) && value.present?
+          # merge this and previous row if nils are present
+          value = merge_row_values( value, collapsed_data[range] ) unless collapsed_data[range].nil?
+          # assign value
+          collapsed_data[range] = value
+        end
       end
       to_table(collapsed_data)
     end
@@ -61,6 +73,17 @@ class Collapse
       end
     end
     
+    def merge_row_values(top_row, bottom_row)
+      # merge previous values when nils are present
+      if top_row.is_a?(Array) && top_row.include?(nil)
+        # find nil indexes
+        indexes = find_each_index(top_row, nil)
+        # merge nils with previous values
+        indexes.each{|index| top_row[index] = bottom_row[index] }
+      end
+      top_row
+    end
+    
     def collapses_greater_than(freq)
       index = valid_collapses.index(freq.to_sym)
       index.present? ? valid_collapses.slice( index + 1, valid_collapses.count ) : []
@@ -82,6 +105,18 @@ class Collapse
       Date.jd(date).end_of_frequency(frequency).jd
     end
   
+    def find_each_index(array, find)
+      found, index, q = -1, -1, []
+      while found
+        found = array[index+1..-1].index(find)
+        if found
+          index = index + found + 1
+          q << index
+        end
+      end
+      q
+    end
+  
   end
 
 end

data/lib/quandl/operation/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Operation
-  VERSION = "0.1.11"
+  VERSION = "0.1.12"
 end
 end

data/quandl_operation.gemspec

@@ -20,6 +20,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rake", "~> 10.0"
   s.add_development_dependency "rspec", "~> 2.13"
   s.add_development_dependency "fivemat", "~> 1.2"
+  s.add_development_dependency "pry"
+  s.add_development_dependency "quandl_data"
 
   s.add_runtime_dependency "activesupport", ">= 3.0.0"
   

data/spec/lib/quandl/operation/collapse_spec.rb

@@ -35,4 +35,68 @@ describe Quandl::Operation::Collapse do
     subject.collapses_greater_than_or_equal_to(:annual).should eq [:annual]
   end
   
+  def table_range(from, to, columns = 1)
+    days = to - from
+    r = days.times.collect do |i|
+      [ from + i ] + columns.times.collect{(rand(1000) * 0.7) + i}
+    end
+    r
+  end
+  
+  describe "#perform" do
+  
+    let(:data){
+      [[2456537, 56.2, nil, nil], 
+       [2456518, 55.7, nil, nil], [2456506, nil, 136133.0, nil], 
+       [2456487, 55.4, nil, nil], [2456475, nil, 135964.0, nil], 
+       [2456457, 50.9, nil, nil], [2456445, nil, 135860.0, nil]]
+    }
+    
+    it "should collapse data with nils" do
+      subject.perform(data, :monthly).should eq [
+        [2456566, 56.2, nil, nil], 
+        [2456536, 55.7, 136133.0, nil], 
+        [2456505, 55.4, 135964.0, nil], 
+        [2456474, 50.9, 135860.0, nil]]
+    end
+   
+  end
+  
+  describe "#collapse" do
+  
+    it 'should handle data sets with one data point only' do
+    
+      data = [[ 2455875, 42 ]]
+      subject.collapse( data, :daily).should eq [[2455875, 42]]
+      subject.collapse( data, :weekly).should eq [[2455879, 42]]
+      subject.collapse( data, :monthly).should eq [[2455896, 42]]
+      subject.collapse( data, :quarterly).should eq [[2455927, 42]]
+      subject.collapse( data, :annual).should eq [[2455927, 42]]
+    
+    end
+  
+    it 'should handle data sets with only two data points, 1 day apart' do
+    
+      data = [[ Date.parse('2011-11-09').jd, 20111109 ],[ Date.parse('2011-11-10').jd, 20111110 ]]
+    
+      subject.collapse(data, :daily).should eq data
+      subject.collapse(data, :weekly).should eq [[ 2455879, 20111110 ]]
+      subject.collapse(data, :monthly).should eq [[ 2455896, 20111110 ]]
+      subject.collapse(data, :quarterly).should eq [[ 2455927, 20111110 ]]
+      subject.collapse(data, :annual).should eq [[ 2455927, 20111110 ]]
+    
+    end
+  
+    it 'should handle data sets one year of daily data' do
+      data = table_range Date.parse('Jan 1, 2011').jd, Date.parse('Dec 31, 2011').jd, 2
+      data.count.should eq 364
+      subject.collapse( data, :daily ).count.should eq 364
+      subject.collapse( data, :weekly ).count.should eq 53
+      subject.collapse( data, :monthly ).count.should eq 12
+      subject.collapse( data, :quarterly ).count.should eq 4
+      subject.collapse( data, :annual ).count.should eq 1
+    end
+   
+  end
+  
 end

data/spec/spec_helper.rb

@@ -1,4 +1,5 @@
 $:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
 
 require "rspec"
-require "quandl/operation"
+require "quandl/operation"
+require 'pry'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: quandl_operation
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-30 00:00:00.000000000 Z
+date: 2013-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -59,6 +59,38 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: quandl_data
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement