quandl_operation 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 080bea06efe1f790ddb4d1e46750669198add637
+  data.tar.gz: 17ee6fd001ebc3d5fb4567e2ffb7bd14b489caf3
+SHA512:
+  metadata.gz: 75d3ebbf079378077b588b5583eb38503a562bb4fd055ffb2e16aafda476a890f32900dc06ffdb7070e9ca4cc2e29836fcfef28c09604e404e1a78e3023a3817
+  data.tar.gz: 27ce390435c47961e805a31e5f3b27d2e8e13c1c015ae3b308b95664462751e7b6879ef9a9524533484c502aa9dad5eb4ef1720401e00ac52c7beb7ba6f203a8

data/.gitignore

@@ -0,0 +1,4 @@
+/Gemfile.lock
+/pkg
+/tmp
+quandl_operation-*

data/.rspec

@@ -0,0 +1 @@
+--colour --format=Fivemat

data/.travis.yml

@@ -0,0 +1,20 @@
+language: ruby
+
+rvm:
+  - 2.0.0
+  - 1.9.3
+  - 1.9.2
+  - 1.8.7
+
+gemfile:
+  - Gemfile
+  - gemfiles/activemodel-4.0.gemfile
+
+matrix:
+  exclude:
+  - rvm: 1.8.7
+    gemfile: gemfiles/activemodel-4.0.gemfile
+  - rvm: 1.9.2
+    gemfile: gemfiles/activemodel-4.0.gemfile
+
+script: "echo 'COME ON!' && bundle exec rake spec"

data/.yardopts

@@ -0,0 +1,2 @@
+--protected
+--no-private

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2012-2013 Blake Hilscher
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,15 @@
+# Quandl::Operation
+
+```ruby
+
+data = [[2456461, 1, 2, 3],[2456460, 2, 3, 4]]
+
+Quandl::Operation::Collapse.perform(data, :weekly)
+Quandl::Operation::Transform.perform(data, :rdiff)
+
+
+csv = "2456461,9.992941176470588,10.974117647058824\n2456459,9.960611072664358,10.92053813148789\n"
+
+Quandl::Operation::Parse.perform(csv)
+
+```

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler"
+require "rake"
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+task :default => :spec
+
+desc "Run all specs"
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.pattern = "spec/**/*_spec.rb"
+end

data/UPGRADE.md

@@ -0,0 +1,3 @@
+## 0.1
+
+Initial

data/lib/quandl/operation.rb

@@ -0,0 +1,17 @@
+require "quandl/operation/version"
+
+require "active_support"
+require "active_support/inflector"
+require "active_support/core_ext/hash"
+require "active_support/core_ext/object"
+
+require 'quandl/operation/core_ext'
+require 'quandl/operation/common_logger'
+require 'quandl/operation/collapse'
+require 'quandl/operation/transform'
+require 'quandl/operation/parse'
+
+module Quandl
+  module Operation
+  end
+end

data/lib/quandl/operation/collapse.rb

@@ -0,0 +1,57 @@
+# subclasses
+require 'quandl/operation/collapse/guess'
+# collapse
+module Quandl
+module Operation
+
+class Collapse
+
+  class << self
+  
+    def perform(data, frequency)
+      data = Parse.sort( data )
+      data = collapse_and_log(data, frequency)
+      data
+    end
+    
+    def collapse_and_log(*args)
+      t1 = Time.now
+      r = collapse(*args)
+      CommonLogger.debug "#{self.name}.perform (#{t1.elapsed.microseconds}ms)"
+      r
+    end
+  
+    def collapse(data, frequency)
+      # store the new collapsed data
+      collapsed_data = {}
+      range = find_end_of_range( data[0][0], frequency )
+      # iterate over the data
+      data.each do |row|
+        # grab date and value
+        date, value = row[0], row[1..-1]
+        value = value.first if value.count == 1
+        # bump to the next range if it exceeds the current one
+        range = find_end_of_range(date, frequency) unless inside_range?(date, range)
+        # consider the value for the next range
+        collapsed_data[range] = value if inside_range?(date, range) && value.present?
+      end
+      collapsed_data
+    end
+  
+    def frequency?(data)
+      Guess.frequency(data)
+    end
+  
+    def inside_range?(date, range)
+      date <= range
+    end
+  
+    def find_end_of_range(date, frequency)
+      Date.jd(date).end_of_frequency(frequency).jd
+    end
+  
+  end
+
+end
+end
+end

data/lib/quandl/operation/collapse/guess.rb

@@ -0,0 +1,93 @@
+module Quandl
+module Operation
+class Collapse
+
+class Guess
+  class << self
+  
+    def frequency(data)
+      return :annual unless data && data[0] && data[0][0]
+      t1 = Time.now
+      # find the smallest point of difference between dates
+      gap = find_average_gap(data)
+      # ensure gap is not negative
+      gap = ensure_positive_gap(gap)
+      # determine the freq from the size of the smallest gap
+      freq = frequency_from_gap(gap)
+      CommonLogger.debug "#{self.name}.perform (#{t1.elapsed.microseconds}ms)"
+      freq
+    end
+  
+    def find_smallest_gap(data)
+      # init
+      gap = 100_000
+      pdate = nil
+      # find the smallest gap
+      data.each do |row|
+        # if the gap is 1, we're done
+        break if gap <= 1
+        # this row's date
+        date = row[0]
+        # only if pdate is present
+        if pdate
+          # calculate the gap
+          diff = (pdate - date).to_i
+          # replace the previous gap if it is smaller
+          gap = diff if diff < gap
+        end
+        # previous row's date
+        pdate = date
+      end
+      gap
+    end
+  
+    def find_average_gap(data)
+      # init
+      gap = 100_000
+      pdate = nil
+      row_count = data.count
+      majority_count = (row_count * 0.55).to_i
+      gaps = {}
+      # find the smallest gap
+      data.each do |row|
+        # this row's date
+        date = row[0]
+        # only if pdate is present
+        if pdate
+          # calculate the gap
+          diff = (pdate - date).to_i
+          # increment the gap counter
+          gaps[diff] ||= 0
+          gaps[diff] += 1
+          # if the diff count is greater than majority_count, we have a consensus
+          return diff if gaps[diff] > majority_count
+        end
+        # previous row's date
+        pdate = date
+      end
+      gaps.to_a.sort_by{|r| r[1] }.try(:last).try(:first)
+    end
+
+    def frequency_from_gap(gap)
+    
+      case 
+      when gap <= 1 then :daily
+      when gap <= 10 then :weekly
+      when gap <= 31 then :monthly
+      when gap <= 93 then :quarterly
+      else :annual
+      end
+    end
+  
+    def ensure_positive_gap(gap)
+      gap = gap.to_i
+      gap = gap * -1 if gap < 0
+      gap
+    end
+  
+  end
+  
+end
+end
+end
+end

data/lib/quandl/operation/common_logger.rb

@@ -0,0 +1,21 @@
+module Quandl
+module Operation
+  
+class CommonLogger
+  
+  class << self
+  
+    delegate :info, :debug, :with_time_elapsed, to: :logger, allow_nil: true
+  
+    def logger
+      @@logger if defined?(@@logger)
+    end
+    def use(value)
+      @@logger = value
+    end
+  
+  end
+  
+end
+end
+end

data/lib/quandl/operation/core_ext.rb

@@ -0,0 +1,4 @@
+require 'quandl/operation/core_ext/array'
+require 'quandl/operation/core_ext/date'
+require 'quandl/operation/core_ext/time'
+require 'quandl/operation/core_ext/float'

data/lib/quandl/operation/core_ext/array.rb

@@ -0,0 +1,5 @@
+class Array
+  def self.forwardable_methods
+    [:reject, :keep_if, :permutation, :to_a, :cycle, :drop, :take_while, :map, :rotate, :each_slice, :pack, :select!, :combination, :repeated_combination, :shift, :select, :reverse!, :==, :clear, :rotate!, :inspect, :iter_for_each, :sort_by!, :compact!, :|, :copy_data_simple, :nitems, :zip, :take, :rassoc, :flatten!, :join, :compact, :[]=, :frozen?, :slice!, :drop_while, :reverse_each, :shuffle, :slice, :reverse, :insert, :uniq, :first, :count, :fetch, :hash, :to_ary, :find_index, :replace, :-, :product, :iter_for_reverse_each, :pop, :push, :sort, :fill, :uniq!, :length, :&, :flatten, :repeated_permutation, :[], :shuffle!, :sort!, :sample, :include?, :<<, :dimensions, :collect, :+, :rindex, :<=>, :eql?, :indices, :collect!, :iter_for_each_index, :iter_for_each_with_index, :index, :*, :indexes, :copy_data, :delete, :to_s, :assoc, :delete_at, :unshift, :delete_if, :empty?, :reject!, :last, :size, :concat, :map!, :at, :each_index, :transpose, :values_at, :each, :enum_cons, :group_by, :enum_with_index, :entries, :with_object, :chunk, :each_with_index, :min, :inject, :one?, :partition, :enum_slice, :none?, :max_by, :any?, :flat_map, :reduce, :each_entry, :find, :minmax_by, :collect_concat, :each_cons, :member?, :max, :sort_by, :detect, :all?, :minmax, :grep, :each_with_object, :find_all]
+  end
+end

data/lib/quandl/operation/core_ext/date.rb

@@ -0,0 +1,25 @@
+class Date
+  
+  def start_of_frequency(freq)
+    case freq.to_sym
+    when :daily     then self
+    when :weekly    then self.beginning_of_week
+    when :monthly   then self.beginning_of_month
+    when :quarterly then self.beginning_of_quarter
+    when :annual    then self.beginning_of_year
+    when :annually  then self.beginning_of_year
+    end
+  end
+  
+  def end_of_frequency(freq)
+    case freq.to_sym
+    when :daily     then self
+    when :weekly    then self.end_of_week
+    when :monthly   then self.end_of_month
+    when :quarterly then self.end_of_quarter
+    when :annual    then self.end_of_year
+    when :annually  then self.end_of_year
+    end
+  end
+  
+end

data/lib/quandl/operation/core_ext/float.rb

@@ -0,0 +1,7 @@
+class Float
+
+  def microseconds
+    (self * 1000.0).to_i
+  end
+  
+end

data/lib/quandl/operation/core_ext/time.rb

@@ -0,0 +1,19 @@
+class Time
+  
+  def microseconds
+    (self.to_f * 1000.0).to_i
+  end
+  
+  def elapsed
+    elapsed_since(Time.now)
+  end
+  
+  def elapsed_since(time)
+    time - self
+  end
+  
+  def elapsed_ms
+    "#{elapsed.microseconds}ms"
+  end
+  
+end

data/lib/quandl/operation/parse.rb

@@ -0,0 +1,96 @@
+module Quandl
+module Operation
+  
+class Parse
+  
+  class << self
+  
+    def perform(data)
+      return [] if data.blank?
+      t1 = Time.now
+      data = csv(data)
+      data = julian_string_to_integer(data)
+      data = sort(data)
+      CommonLogger.debug "#{self.name}.perform (#{t1.elapsed.microseconds}ms)"
+      data
+    end
+  
+    def csv(data)
+      data = CSV.parse( data ) if data.is_a?(String)
+      data
+    end
+  
+    def sort(data, order = :asc)
+      data_order = sort_order?(data)
+      # ascending
+      if order == :asc  && data_order != :asc
+        data = sort_asc(data)
+        # descending
+      elsif order == :desc && data_order != :desc
+        data = sort_desc(data)
+      end
+      data
+    end
+  
+    def sort_order?(data)
+      return :none if data.blank? || data[0].blank? || data[1].blank?
+      data[0][0] > data[1][0] ? :desc : :asc
+    end
+  
+    def sort_asc(data)
+      data.sort_by{|r| r[0] }
+    end
+  
+    def sort_desc(data)
+      data.sort_by{|r| r[0] }.reverse
+    end
+  
+    def date_to_julian(data)
+      return data if data[0][0].is_a?(Integer)
+      # dont alter by reference
+      result = []
+      # for each row
+      data.each_with_index do |row, index|
+        # copy
+        nrow = row.dup
+        # string to date
+        date = nrow[0]
+        date = Date.parse(nrow[0]) if nrow[0].is_a?(String)
+        # date to julian
+        nrow[0] = date.jd if date.respond_to?(:jd)
+        # save result
+        result[index] = nrow
+      end
+      # all done
+      result
+    end
+  
+    def julian_to_date(data)
+      return data if data[0][0].is_a?(Date)
+      # dont alter by reference
+      result = []
+      # for each row
+      data.each_with_index do |row, index|
+        # copy
+        nrow = row.dup
+        # parse date
+        nrow[0] = Date.jd( nrow[0].to_i ) unless nrow[0].is_a?(Date)
+        # save result
+        result[index] = nrow
+      end
+      # all done
+      result
+    end
+  
+    def julian_string_to_integer(data)
+      # skip when already formatted correctly
+      return data if data[0][0].is_a?(Integer) || data[0][0].is_a?(Date)
+      # otherwise cast string jds to int
+      data.collect{|r| r[0] = r[0].to_i; r  }
+    end
+  
+  end
+  
+end
+end
+end

data/lib/quandl/operation/transform.rb

@@ -0,0 +1,131 @@
+# collapse
+module Quandl
+module Operation
+  
+class Transform
+  class << self
+
+    def perform( data, type)
+      data = Parse.sort( data, :asc )
+      data = transform_and_log( data, type)
+      data
+    end
+    
+    def transform_and_log(*args)
+      t1 = Time.now
+      r = transform(*args)
+      CommonLogger.debug "#{self.name}.perform (#{t1.elapsed.microseconds}ms)"
+      r
+    end
+    
+    def valid_transformation?(type)
+      valid_transformations.include?( type )
+    end
+    
+    def valid_transformations
+      [ :diff, :rdiff, :cumul, :normalize ]
+    end
+  
+    def transform( data, type)
+      return data if data.blank?
+      #Transforms table from actual data points
+      #to differences between points (:diff)
+      #or a ratio between points(:rdiff)
+      #If type is other than these two, nothing is done.
+    
+      # ensure that type is in the expected format
+      type = type.to_sym
+      # nothing to do unless valid transform
+      return unless valid_transformation?( type )
+    
+      temparr = Array.new
+      #first make a keylist
+      keylist = data.transpose.first
+      # now sort the keylist from oldest to newest
+      # unless there is only one point
+      if keylist.count > 1
+        keylist = keylist.reverse if keylist[0] > keylist[1] # better performance if we do this first
+        keylist.sort!
+      end
+
+      #find number of columns
+      numcols = data.first.size - 1
+      if type == :normalize
+        divisor = Array.new(numcols,nil)
+        0.upto(keylist.length - 1) do |i|
+          temparr[i] = []
+          curr_row = data[i][1..-1]
+          0.upto(numcols-1) do |x|
+            if curr_row[x].nil?
+              temparr[i][x] = nil
+            elsif divisor[x].nil?
+              if curr_row[x].to_f != 0 
+                divisor[x] = curr_row[x].to_f
+                temparr[i][x] = 100.0
+              else
+                temparr[i][x] = 0
+              end
+            else
+              temparr[i][x] = curr_row[x] / divisor[x] * 100.0
+            end
+          end
+        end
+        0.upto(keylist.length-1) do |i|
+          data[i] = [keylist[i], temparr[i]].flatten
+        end
+      elsif [:diff, :rdiff].include? type
+        #now build temparr
+        1.upto(keylist.length - 1) do |i|
+          temparr[i] = []
+          curr_row = data[i][1..-1]
+          prev_row = data[i-1][1..-1]
+          0.upto(numcols-1) do |x| 
+            if type == :diff
+              if !curr_row[x].nil? and !prev_row[x].nil?
+                temparr[i][x] = Float(curr_row[x]) - Float(prev_row[x])
+              else
+                temparr[i][x] = nil
+              end
+            else
+              if !curr_row[x].nil? and !prev_row[x].nil? and prev_row[x] != 0
+                temparr[i][x] = ( Float(curr_row[x]) - Float(prev_row[x]) ) / Float( prev_row[x] )
+              else
+                temparr[i][x] = nil
+              end
+            end
+          end
+        end
+
+        #now put temparr into datapac
+        1.upto(keylist.length-1) do |i|
+          data[i] = [keylist[i], temparr[i]].flatten
+        end
+
+        #delete the first date in datapac (because there is no diff for that)
+        data.delete_at(0)
+      else
+        cumulsum = Array.new(numcols,0)
+        sumstarted = Array.new(numcols,false)
+        #now build temparr
+        0.upto(keylist.length - 1) do |i|
+          temparr[i] = []
+          curr_row = data[i][1..-1]
+          0.upto(numcols-1) do |x|
+            if !curr_row[x].nil?
+              cumulsum[x] = cumulsum[x] + curr_row[x]
+              sumstarted[x] = true
+            end
+            temparr[i][x] = cumulsum[x] if sumstarted[x]
+          end
+        end
+        0.upto(keylist.length-1) do |i|
+          data[i] = [keylist[i],temparr[i]].flatten
+        end
+      end
+      data
+    end
+    
+  end
+end
+end
+end

data/lib/quandl/operation/version.rb

@@ -0,0 +1,5 @@
+module Quandl
+module Operation
+  VERSION = "0.0.1"
+end
+end

data/quandl_operation.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "quandl/operation/version"
+
+Gem::Specification.new do |s|
+  s.name        = "quandl_operation"
+  s.version     = Quandl::Operation::VERSION
+  s.authors     = ["Blkae Hilscher"]
+  s.email       = ["blake@hilscher.ca"]
+  s.homepage    = "http://blake.hilscher.ca/"
+  s.license     = "MIT"
+  s.summary     = "For altering data"
+  s.description = "Data will be operated"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rake", "~> 10.0"
+  s.add_development_dependency "rspec", "~> 2.13"
+  s.add_development_dependency "fivemat", "~> 1.2"
+
+  s.add_runtime_dependency "activesupport", ">= 3.0.0"
+end

data/spec/quandl_operation_collapse_spec.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Operation::Collapse do
+  
+  it { should respond_to :perform }
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+
+require "rspec"
+require "scope_builder"

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification
+name: quandl_operation
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Blkae Hilscher
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.13'
+- !ruby/object:Gem::Dependency
+  name: fivemat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+description: Data will be operated
+email:
+- blake@hilscher.ca
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- .yardopts
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- UPGRADE.md
+- lib/quandl/operation.rb
+- lib/quandl/operation/collapse.rb
+- lib/quandl/operation/collapse/guess.rb
+- lib/quandl/operation/common_logger.rb
+- lib/quandl/operation/core_ext.rb
+- lib/quandl/operation/core_ext/array.rb
+- lib/quandl/operation/core_ext/date.rb
+- lib/quandl/operation/core_ext/float.rb
+- lib/quandl/operation/core_ext/time.rb
+- lib/quandl/operation/parse.rb
+- lib/quandl/operation/transform.rb
+- lib/quandl/operation/version.rb
+- quandl_operation.gemspec
+- spec/quandl_operation_collapse_spec.rb
+- spec/spec_helper.rb
+homepage: http://blake.hilscher.ca/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: For altering data
+test_files:
+- spec/quandl_operation_collapse_spec.rb
+- spec/spec_helper.rb