quandl_babelfish 0.0.9 → 0.0.10

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler"
+require "rake"
+require "bundler/gem_tasks"
+
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+
+require 'pry'
+require "quandl/babelfish"

data/lib/quandl/babelfish/date_maid.rb

@@ -49,10 +49,10 @@ module Babelfish
         # Regular formats and Custom formats (where Date.parse and Date.strptime
         # fear to tread)
         if re = example.match(/^(\d{1,2})\D(\d{1,2})\D\d{4}/) # eg "07/03/2012"
-          if re[1].to_i > 12
-            return '%d-%m-%Y', nil
-          else
+          if re[2].to_i > 12
             return '%m-%d-%Y', nil
+          else
+            return '%d-%m-%Y', nil
           end
         end
         if re = example.match(/^(\d{1,2})\D(\d{1,2})\D\d{2}/) # eg "07/03/12"
@@ -173,27 +173,36 @@ module Babelfish
 
 
       #find good example of date to use as template for format
-      def find_good_date(all_dates)
+      #if strict == true, no ambiguity is tolerated.  If strict= false, we will accept abbiguity.  (02/05/2009)
+      def find_good_date(all_dates, strict=true)
         good_sample=nil
         all_dates.each do |fuzzy_date|
-          if  usable_cell(fuzzy_date)
+          if  usable_cell(fuzzy_date,strict)
             good_sample = fuzzy_date
             break
           end
         end
-        good_sample
+        if good_sample == nil and strict==true
+          # We could not find a single unambiguous cell.  Let's now be less strict and see if we can find something
+          find_good_date(all_dates,false)
+        else
+          good_sample
+        end
       end
 
-      def usable_cell(cell)
+      # if strict == true then we refuse to accept any ambiguity
+      # if strict == false, we'll settle for a bit of ambiguity
+      def usable_cell(cell,strict)
         return false if cell.nil? || cell.to_s.empty?
         return false if cell.to_s.size > 20 # even annotated date can't be bigger than 20
 
         return true if cell.to_s =~ /^\w{3}\D[456789]\d$/
+
         # date is not usable as an example if it is ambiguous as to day and month
         # 03/04/2012, for example, is ambiguous.  03/17/2012 is NOT ambiguous
         if re = cell.to_s.match(/^(\d{1,2})\D(\d{1,2})\D\d{2,4}/)  # e.g. 03/04/2012
           if re[1].to_i <= 12 and re[2].to_i <= 12
-            return false
+            return strict==true ? false : true
           else
             return true
           end

data/lib/quandl/babelfish/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Babelfish
-  VERSION = '0.0.9'
+  VERSION = '0.0.10'
 end
 end

data/quandl_babelfish.gemspec

@@ -18,4 +18,5 @@ Gem::Specification.new do |s|
   
   s.add_development_dependency "rspec", "~> 2.13"
   s.add_development_dependency "pry"
+  s.add_development_dependency "rake"
 end

data/spec/lib/quandl/babelfish/cleaner_spec.rb

@@ -13,6 +13,11 @@ describe Cleaner do
     let(:input){ [[2012, nil], [2011, 20], [2010, 30]] }
     it{ should eq [[Date.new(2012,12,31), nil], [Date.new(2011,12,31), 20.0], [Date.new(2010,12,31), 30.0]] }
   end
+
+  context "given nil" do
+    let(:input){ [[2002,'#N.A.'], [2011, 20]]}
+    it{ should eq [[Date.new(2002,12,31), nil], [Date.new(2011,12,31), 20.0]] }
+  end
   
   context "mismatch row count" do
     let(:input){ [[2012], [2011, 20], [2010, 30, 25]] }

data/spec/lib/quandl/babelfish/date_maid_spec.rb

@@ -3,16 +3,6 @@ require 'spec_helper'
 include Quandl::Babelfish
 describe NumberMaid do
 
-  it 'should return an exception because month and day are ambiguous YYYY' do
-    dates = ['01/01/2011','1/2/2011','2/3/2011','11/1/2011']
-    lambda {DateMaid::sweep(dates)}.should raise_error(Error::GuessDateFormat)
-  end
-
-  it 'should return an exception because month and day are ambiguous YY' do
-    dates = ['01/01/11','1/2/11','1/3/11','11/1/11']
-    lambda {DateMaid::sweep(dates)}.should raise_error(Error::GuessDateFormat)
-  end
-
   it 'should remove unwanted characters from dates (eg.  )' do
     a=194.chr+160.chr
     dates = ["2005#{a}","#{a}2006",'2007','2008']
@@ -525,5 +515,14 @@ describe NumberMaid do
     dates[1].should == Date.new(2012,04,29)
   end
 
+  it 'should handle US format even if it is ambiguous' do
+    dates = ['1/1/1954','1/4/1954','1/7/1954','1/11/1954']
+    dates  = DateMaid::sweep(dates)
+    dates[0].should == Date.new(1954,1,1)
+    dates[1].should == Date.new(1954,4,1)
+    dates[2].should == Date.new(1954,7,1)
+    dates[3].should == Date.new(1954,11,1)
+  end
+
 
 end

data/spec/lib/quandl/babelfish/number_maid_spec.rb

@@ -123,4 +123,9 @@ describe NumberMaid do
     numbers.should == 0.12345678901235
   end
 
+  it "should handle pound in front" do
+    NumberMaid::clean('#N/A').should be_nil
+  end
+
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: quandl_babelfish
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-10 00:00:00.000000000 Z
+date: 2014-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -43,6 +43,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Quandl Data Cleaner
 email:
 - sergei@quandl.com
@@ -55,6 +71,7 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- Rakefile
 - UPGRADE.md
 - lib/quandl/babelfish.rb
 - lib/quandl/babelfish/chronometer.rb
@@ -89,12 +106,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 4375924718490433472
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 4375924718490433472
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23