quandl_babelfish 0.0.7 → 0.0.8

data/lib/quandl/babelfish/helper.rb

@@ -1,9 +1,9 @@
-class Helper
-
-  # Actions expect a square table, make it so
-  def self.make_square(table)
-    longest_row = 0
-    table.each { |row| longest_row = [longest_row, row.length].max }
-    table.collect { |row| row += Array.new(longest_row - row.length, nil) }
-  end
+class Helper
+
+  # Actions expect a square table, make it so
+  def self.make_square(table)
+    longest_row = 0
+    table.each { |row| longest_row = [longest_row, row.length].max }
+    table.collect { |row| row += Array.new(longest_row - row.length, nil) }
+  end
 end

data/lib/quandl/babelfish/number_maid.rb

@@ -1,80 +1,80 @@
-module Quandl
-module Babelfish
-
-  #responsible for number cleaning
-  class NumberMaid
-    @defaults = {
-        :decimal_mark => Regexp.escape('.'),
-        :ignore_brackets => false, # Brackets ARE negative by default
-    }
-
-    @settings = @defaults   #init with defaults
-
-    class << self
-
-
-      def init(user_settings)
-        @settings=@defaults.merge(user_settings)
-        @escaped_decimal = Regexp.escape(@settings[:decimal_mark])
-      end
-
-      #cleans each number one by one
-      def clean(dirty_numbers)
-        return nil if dirty_numbers.nil?
-        numbers=[]
-        Array(dirty_numbers).each do |cell|
-          numbers << cell_to_number(cell.to_s)
-        end
-
-        (numbers.size == 1) ? numbers[0] : numbers
-      end
-
-      def cell_to_number(num)
-        return nil if num.nil?
-        # Remove annotations
-        # if there is something in parenthesis and a number elsewhere, nuke the parenthesis
-        temp = num.gsub(/[\(\[\{].*[\)\}\]]/, '')
-        num = temp if temp.match(/\d/)
-
-        num.gsub!("est.", '')
-
-        #check for exponents by searching for 'e' 'E' or variations of 'x 10' '*10' and 'X10^'
-        is_exp = false
-        expmultiplier = 1
-        m = /(\s)*(E|e|[X|x|\*](\s)*10(\^)?)(\s)*/.match(num)
-        #check if match is made, preceeded by a number/decimal, and succeeded by a digit or a plus/minus sign
-        if !m.nil? and m.pre_match =~ /[0-9#{@escaped_decimal}]$/ and m.post_match =~ /^([\-+0-9])/
-          is_exp = true
-          num = m.pre_match
-          expmultiplier = 10 ** /^[0-9\-+]*/.match(m.post_match)[0].to_i
-        end
-        is_million = (num =~ /million/i)
-        is_billion = (num =~ /billion/i)
-        is_negative = (num =~ /-[\d]/ or (!@settings[:ignore_brackets] and num =~ /\([\d]/))
-
-        # watch out for two numbers, like a range  eg "27.3 - 33.9"
-        # how: if you a see a number followed by a non number char that is not the decimal marker, kill everything to the right of that
-        num.gsub!(/(\d) (\d)/, '\1\2')
-        if m = num.match(/-?\s*[,\d\.]+/)
-          num = m[0]
-        end
-
-        # only keep #s and decimal mark
-        num.gsub!(/[^0-9#{@escaped_decimal}]/, '')
-        num.gsub!(/[^0-9]/, '.')
-
-        return nil if num.nil? || num !~ /[\d]/
-        return nil if num.end_with?(".")
-        return nil if num.count(".") > 1
-        cell = num.nil? ? 0.0 : Float("%.#{14}g" % num)
-        cell *= 1e6 if is_million
-        cell *= 1e9 if is_billion
-        cell *= -1 if is_negative
-        cell *= expmultiplier if is_exp
-        cell
-      end
-
-    end
-  end
-end
+module Quandl
+module Babelfish
+
+  #responsible for number cleaning
+  class NumberMaid
+    @defaults = {
+        :decimal_mark => Regexp.escape('.'),
+        :ignore_brackets => false, # Brackets ARE negative by default
+    }
+
+    @settings = @defaults   #init with defaults
+
+    class << self
+
+
+      def init(user_settings)
+        @settings=@defaults.merge(user_settings)
+        @escaped_decimal = Regexp.escape(@settings[:decimal_mark])
+      end
+
+      #cleans each number one by one
+      def clean(dirty_numbers)
+        return nil if dirty_numbers.nil?
+        numbers=[]
+        Array(dirty_numbers).each do |cell|
+          numbers << cell_to_number(cell.to_s)
+        end
+
+        (numbers.size == 1) ? numbers[0] : numbers
+      end
+
+      def cell_to_number(num)
+        return nil if num.nil?
+        # Remove annotations
+        # if there is something in parenthesis and a number elsewhere, nuke the parenthesis
+        temp = num.gsub(/[\(\[\{].*[\)\}\]]/, '')
+        num = temp if temp.match(/\d/)
+
+        num.gsub!("est.", '')
+
+        #check for exponents by searching for 'e' 'E' or variations of 'x 10' '*10' and 'X10^'
+        is_exp = false
+        expmultiplier = 1
+        m = /(\s)*(E|e|[X|x|\*](\s)*10(\^)?)(\s)*/.match(num)
+        #check if match is made, preceeded by a number/decimal, and succeeded by a digit or a plus/minus sign
+        if !m.nil? and m.pre_match =~ /[0-9#{@escaped_decimal}]$/ and m.post_match =~ /^([\-+0-9])/
+          is_exp = true
+          num = m.pre_match
+          expmultiplier = 10 ** /^[0-9\-+]*/.match(m.post_match)[0].to_i
+        end
+        is_million = (num =~ /million/i)
+        is_billion = (num =~ /billion/i)
+        is_negative = (num =~ /-[\d]/ or (!@settings[:ignore_brackets] and num =~ /\([\d]/))
+
+        # watch out for two numbers, like a range  eg "27.3 - 33.9"
+        # how: if you a see a number followed by a non number char that is not the decimal marker, kill everything to the right of that
+        num.gsub!(/(\d) (\d)/, '\1\2')
+        if m = num.match(/-?\s*[,\d\.]+/)
+          num = m[0]
+        end
+
+        # only keep #s and decimal mark
+        num.gsub!(/[^0-9#{@escaped_decimal}]/, '')
+        num.gsub!(/[^0-9]/, '.')
+
+        return nil if num.nil? || num !~ /[\d]/
+        return nil if num.end_with?(".")
+        return nil if num.count(".") > 1
+        cell = num.nil? ? 0.0 : Float("%.#{14}g" % num)
+        cell *= 1e6 if is_million
+        cell *= 1e9 if is_billion
+        cell *= -1 if is_negative
+        cell *= expmultiplier if is_exp
+        cell
+      end
+
+    end
+  end
+end
 end

data/lib/quandl/babelfish/version.rb

@@ -1,5 +1,5 @@
-module Quandl
-module Babelfish
-  VERSION = '0.0.7'
-end
+module Quandl
+module Babelfish
+  VERSION = '0.0.8'
+end
 end

data/lib/quandl/error/guess_date_format.rb

@@ -1,5 +1,5 @@
-module Quandl
-module Error
-  class GuessDateFormat < Quandl::Error::Standard; end
-end
+module Quandl
+module Error
+  class GuessDateFormat < Quandl::Error::Standard; end
+end
 end

data/lib/quandl/error/invalid_date.rb

@@ -1,5 +1,5 @@
-module Quandl
-module Error
-  class InvalidDate < Quandl::Error::Standard; end
-end
+module Quandl
+module Error
+  class InvalidDate < Quandl::Error::Standard; end
+end
 end

data/lib/quandl/error/standard.rb

@@ -1,27 +1,27 @@
-module Quandl
-module Error
-  class Standard < StandardError
-    
-    attr_accessor :details
-    
-    def line
-      detail :line
-    end
-    def context
-      detail :context
-    end
-    def problem
-      detail :problem
-    end
-    
-    def detail(key)
-      details.send(key) if details.respond_to?(key)
-    end
-    
-    def initialize(opts=nil)
-      @details = OpenStruct.new( opts ) if opts && opts.is_a?(Hash)
-    end
-    
-  end
-end
+module Quandl
+module Error
+  class Standard < StandardError
+    
+    attr_accessor :details
+    
+    def line
+      detail :line
+    end
+    def context
+      detail :context
+    end
+    def problem
+      detail :problem
+    end
+    
+    def detail(key)
+      details.send(key) if details.respond_to?(key)
+    end
+    
+    def initialize(opts=nil)
+      @details = OpenStruct.new( opts ) if opts && opts.is_a?(Hash)
+    end
+    
+  end
+end
 end

data/lib/quandl/error/unknown_date_format.rb

@@ -1,5 +1,5 @@
-module Quandl
-module Error
-  class UnknownDateFormat < Quandl::Error::Standard; end
-end
+module Quandl
+module Error
+  class UnknownDateFormat < Quandl::Error::Standard; end
+end
 end

data/quandl_babelfish.gemspec

@@ -1,21 +1,21 @@
-# -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "quandl/babelfish/version"
-
-Gem::Specification.new do |s|
-  s.name        = "quandl_babelfish"
-  s.version     = Quandl::Babelfish::VERSION
-  s.authors     = ["Sergei Ryshkevich"]
-  s.email       = ["sergei@quandl.com"]
-  s.homepage    = "http://quandl.com/"
-  s.license     = "MIT"
-  s.summary     = "Quandl Data Cleaner"
-  s.description = "Quandl Data Cleaner"
-
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.require_paths = ["lib"]
-  
-  s.add_development_dependency "rspec", "~> 2.13"
-  s.add_development_dependency "pry"
-end
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "quandl/babelfish/version"
+
+Gem::Specification.new do |s|
+  s.name        = "quandl_babelfish"
+  s.version     = Quandl::Babelfish::VERSION
+  s.authors     = ["Sergei Ryshkevich"]
+  s.email       = ["sergei@quandl.com"]
+  s.homepage    = "http://quandl.com/"
+  s.license     = "MIT"
+  s.summary     = "Quandl Data Cleaner"
+  s.description = "Quandl Data Cleaner"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ["lib"]
+  
+  s.add_development_dependency "rspec", "~> 2.13"
+  s.add_development_dependency "pry"
+end

data/spec/lib/quandl/babelfish/chronometer_spec.rb

@@ -1,51 +1,51 @@
-require 'spec_helper'
-
-include Quandl::Babelfish
-describe Chronometer do
-
-  it 'should calculate frequency = daily' do
-    table = [['2012-01-01','1','2'],['2012-01-02','3','4'],['2012-01-03','5','6']]
-    frequency = Chronometer.process(table)
-    frequency.should == 'daily'
-  end
-
-  it 'should calculate frequency = monthly' do
-    table = [['2012-01-01','1','2'],['2012-02-01','3','4'],['2012-04-01','5','6'],
-             ['2012-04-01','1','2'],['2012-05-01','3','4'],['2012-06-01','5','6']]
-    frequency = Chronometer.process(table)
-    frequency.should == 'monthly'
-  end
-
-  it 'should calculate frequency = quarterly' do
-    table = [['2012-01-01','1','2'],['2012-04-01','3','4'],['2012-07-01','5','6'],
-             ['2012-10-01','1','2'],['2013-01-01','3','4'],['2012-04-01','5','6']]
-    frequency = Chronometer.process(table)
-    frequency.should == 'quarterly'
-  end
-
-  it 'should calculate frequency = quarterly' do
-    table = [['2012-01-01','1','2'],['2012-07-01','3','4'],['2013-01-01','5','6'],
-             ['2013-07-01','1','2']]
-    frequency = Chronometer.process(table)
-    frequency.should == 'quarterly'
-  end
-
-  it 'should calculate frequency = annual' do
-    table = [['2008-01-01','1','2'],['2008-12-01','3','4'],['2010-01-01','5','6'],
-             ['2011-01-01','1','2'],['2013-01-01','5','6']]
-    frequency = Chronometer.process(table)
-    frequency.should == 'annual'
-  end
-
-  it 'should calculate frequency = daily if only one row' do
-    table = [['2010-01-01','1','2']]
-    frequency = Chronometer.process(table)
-    frequency.should == 'daily'
-  end
-
-  it 'should calculate frequency = nil if nil table passed' do
-    frequency = Chronometer.process(nil)
-    frequency.should == nil
-  end
-
+require 'spec_helper'
+
+include Quandl::Babelfish
+describe Chronometer do
+
+  it 'should calculate frequency = daily' do
+    table = [['2012-01-01','1','2'],['2012-01-02','3','4'],['2012-01-03','5','6']]
+    frequency = Chronometer.process(table)
+    frequency.should == 'daily'
+  end
+
+  it 'should calculate frequency = monthly' do
+    table = [['2012-01-01','1','2'],['2012-02-01','3','4'],['2012-04-01','5','6'],
+             ['2012-04-01','1','2'],['2012-05-01','3','4'],['2012-06-01','5','6']]
+    frequency = Chronometer.process(table)
+    frequency.should == 'monthly'
+  end
+
+  it 'should calculate frequency = quarterly' do
+    table = [['2012-01-01','1','2'],['2012-04-01','3','4'],['2012-07-01','5','6'],
+             ['2012-10-01','1','2'],['2013-01-01','3','4'],['2012-04-01','5','6']]
+    frequency = Chronometer.process(table)
+    frequency.should == 'quarterly'
+  end
+
+  it 'should calculate frequency = quarterly' do
+    table = [['2012-01-01','1','2'],['2012-07-01','3','4'],['2013-01-01','5','6'],
+             ['2013-07-01','1','2']]
+    frequency = Chronometer.process(table)
+    frequency.should == 'quarterly'
+  end
+
+  it 'should calculate frequency = annual' do
+    table = [['2008-01-01','1','2'],['2008-12-01','3','4'],['2010-01-01','5','6'],
+             ['2011-01-01','1','2'],['2013-01-01','5','6']]
+    frequency = Chronometer.process(table)
+    frequency.should == 'annual'
+  end
+
+  it 'should calculate frequency = daily if only one row' do
+    table = [['2010-01-01','1','2']]
+    frequency = Chronometer.process(table)
+    frequency.should == 'daily'
+  end
+
+  it 'should calculate frequency = nil if nil table passed' do
+    frequency = Chronometer.process(nil)
+    frequency.should == nil
+  end
+
 end

data/spec/lib/quandl/babelfish/cleaner_spec.rb

@@ -1,70 +1,70 @@
-require 'spec_helper'
-
-include Quandl::Babelfish
-describe Cleaner do
-
-  let(:input){ [] }
-  let(:output){ Cleaner.process(input) }
-  let(:data){ output[0] }
-  let(:headers){ output[1] }
-  subject{ data }
-  
-  context "garbage" do
-    let(:input){ [[2456624, 10], [2456625, 20], [2456626, 30]] }
-    it{ should be_eq_at_index '[0][0]', Date.new(1970,01,29) }
-  end
-  
-  context "headers with whitespace" do
-    let(:input){ [["   Date   ", " C1    ", "C2   ", "    C4"],[1990,1,2,3],[1991,4,5,6]] }
-    it{ headers.should eq ["Date", "C1", "C2", "C4"] }
-  end
-  
-  context "annual" do
-    let(:input){ [[1990,1,2,3],[1991,4,5,6]] }
-    it{ should be_eq_at_index '[0][0]', Date.new(1990,12,31) }
-    it{ should be_eq_at_index '[0][1]', 1 }
-    it{ should be_eq_at_index '[1][0]', Date.new(1991,12,31) }
-    it{ should be_eq_at_index '[1][3]', 6 }
-    it{ headers.should be_nil }
-  end
-  
-  context "numeric date" do
-    let(:input){ [[19900101,'1 [estimate]','2.3 - 4.0','not a number']] }
-    it{ should be_eq_at_index '[0][0]', Date.new(1990,01,01) }
-    it{ should be_eq_at_index '[0][1]', 1 }
-    it{ should be_eq_at_index '[0][2]', 2.3 }
-    it{ should be_eq_at_index '[0][3]', nil }
-    it{ headers.should be_nil }
-  end
-  
-  context "data with headers" do
-    let(:input){ [['Date',0,0,0],[19900101,'1 [estimate]','2.3 - 4.0','not a number']] }
-    it{ should be_eq_at_index '[0][0]', Date.new(1990,01,01) }
-    it{ should be_eq_at_index '[0][1]', 1 }
-    it{ should be_eq_at_index '[0][2]', 2.3 }
-    it{ should be_eq_at_index '[0][3]', nil }
-    it{ headers.should eq ['Date','0','0','0'] }
-  end
-
-  context "data with nil" do
-    let(:input){ [["Date", "Col1"], ["2002", nil], ["2003", "5"]] }
-    it{ should be_eq_at_index '[0][0]', Date.new(2002,12,31) }
-    it{ data[0].length.should ==2}
-    it{ should be_eq_at_index '[0][1]', nil }
-    it{ should be_eq_at_index '[1][0]', Date.new(2003,12,31)  }
-    it{ should be_eq_at_index '[1][1]', 5 }
-    it{ headers.should eq ['Date','Col1'] }
-  end
-
-  context "data with middle nil" do
-    let(:input){ [["Date", "Col1", "Col2"], ["2002", nil, '1'], ["2003", "5", '6']] }
-    it{ should be_eq_at_index '[0][0]', Date.new(2002,12,31) }
-    it{ should be_eq_at_index '[0][1]', nil }
-    it{ should be_eq_at_index '[0][2]', 1}
-    it{ should be_eq_at_index '[1][0]', Date.new(2003,12,31)  }
-    it{ should be_eq_at_index '[1][1]', 5 }
-    it{ should be_eq_at_index '[1][2]', 6 }
-    it{ headers.should eq ['Date','Col1', 'Col2'] }
-  end
-
-end
+require 'spec_helper'
+
+include Quandl::Babelfish
+describe Cleaner do
+
+  let(:input){ [] }
+  let(:output){ Cleaner.process(input) }
+  let(:data){ output[0] }
+  let(:headers){ output[1] }
+  subject{ data }
+  
+  context "garbage" do
+    let(:input){ [[2456624, 10], [2456625, 20], [2456626, 30]] }
+    it{ should be_eq_at_index '[0][0]', Date.new(1970,01,29) }
+  end
+  
+  context "headers with whitespace" do
+    let(:input){ [["   Date   ", " C1    ", "C2   ", "    C4"],[1990,1,2,3],[1991,4,5,6]] }
+    it{ headers.should eq ["Date", "C1", "C2", "C4"] }
+  end
+  
+  context "annual" do
+    let(:input){ [[1990,1,2,3],[1991,4,5,6]] }
+    it{ should be_eq_at_index '[0][0]', Date.new(1990,12,31) }
+    it{ should be_eq_at_index '[0][1]', 1 }
+    it{ should be_eq_at_index '[1][0]', Date.new(1991,12,31) }
+    it{ should be_eq_at_index '[1][3]', 6 }
+    it{ headers.should be_nil }
+  end
+  
+  context "numeric date" do
+    let(:input){ [[19900101,'1 [estimate]','2.3 - 4.0','not a number']] }
+    it{ should be_eq_at_index '[0][0]', Date.new(1990,01,01) }
+    it{ should be_eq_at_index '[0][1]', 1 }
+    it{ should be_eq_at_index '[0][2]', 2.3 }
+    it{ should be_eq_at_index '[0][3]', nil }
+    it{ headers.should be_nil }
+  end
+  
+  context "data with headers" do
+    let(:input){ [['Date',0,0,0],[19900101,'1 [estimate]','2.3 - 4.0','not a number']] }
+    it{ should be_eq_at_index '[0][0]', Date.new(1990,01,01) }
+    it{ should be_eq_at_index '[0][1]', 1 }
+    it{ should be_eq_at_index '[0][2]', 2.3 }
+    it{ should be_eq_at_index '[0][3]', nil }
+    it{ headers.should eq ['Date','0','0','0'] }
+  end
+
+  context "data with nil" do
+    let(:input){ [["Date", "Col1"], ["2002", nil], ["2003", "5"]] }
+    it{ should be_eq_at_index '[0][0]', Date.new(2002,12,31) }
+    it{ data[0].length.should ==2}
+    it{ should be_eq_at_index '[0][1]', nil }
+    it{ should be_eq_at_index '[1][0]', Date.new(2003,12,31)  }
+    it{ should be_eq_at_index '[1][1]', 5 }
+    it{ headers.should eq ['Date','Col1'] }
+  end
+
+  context "data with middle nil" do
+    let(:input){ [["Date", "Col1", "Col2"], ["2002", nil, '1'], ["2003", "5", '6']] }
+    it{ should be_eq_at_index '[0][0]', Date.new(2002,12,31) }
+    it{ should be_eq_at_index '[0][1]', nil }
+    it{ should be_eq_at_index '[0][2]', 1}
+    it{ should be_eq_at_index '[1][0]', Date.new(2003,12,31)  }
+    it{ should be_eq_at_index '[1][1]', 5 }
+    it{ should be_eq_at_index '[1][2]', 6 }
+    it{ headers.should eq ['Date','Col1', 'Col2'] }
+  end
+
+end