quandl_babelfish 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5aa7214cd7f3c0541f0aefe31d983622065bea21
+  data.tar.gz: 442699776fdbd64183d877e962d564ce2c770664
+SHA512:
+  metadata.gz: 5a577f380ce3771869c553ebb418c82aa5e9dccb1f493e5cfd06f93a3d804c102df3cd6bb38039a5ddc2630d7ed132c4e68fbb6186b60dab0492f73099bd06cf
+  data.tar.gz: ea13348aa261380045206aacadda5c8ef4915f7ea9c9210c00f17d1c75f41bee90ba243f240da73e0431c8a573e9ab42354fe2f55e91732f464c2beb7d0663d8

data/.gitignore

@@ -0,0 +1,7 @@
+.idea/
+/Gemfile.lock
+/pkg
+/tmp
+.rvmrc
+*.gem
+*.log

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+
+rvm:
+  - 2.0.0
+  - 1.9.3
+
+gemfile:
+  - Gemfile
+
+matrix:
+
+script: bundle exec rspec

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2012-2013 Quandl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,18 @@
+# Quandl::Babelfish
+
+### Installation
+
+```ruby
+
+
+gem 'quandl_babelfish'
+
+```
+
+
+### Usage
+
+```ruby
+
+
+```

data/UPGRADE.md

@@ -0,0 +1,6 @@
+## 0.0.1
+
+* replace Cleaner.process return clean_array, header with Quandl::Babelfish::Data.new( clean_array, headers: header )
+* refactored error
+* added header extraction support
+* init

data/lib/quandl/babelfish/cleaner.rb

@@ -0,0 +1,33 @@
+module Quandl
+module Babelfish
+    
+class Cleaner
+  class << self
+    def process(dirty_array, date_settings={}, number_settings={})
+      return nil,nil if dirty_array.nil?
+
+      #check if first line is header
+      header=DateMaid::disinfect(dirty_array.shift) unless DateMaid::analyze_date_format(DateMaid::disinfect(dirty_array[0][0])[0])[0]
+
+      #converts dates first
+      dirty_array
+      dates = dirty_array.collect{|x| x[0]}
+      DateMaid::init(date_settings)
+      clean_dates=DateMaid::sweep dates
+
+      clean_array=[]
+      #clean numbers later
+      NumberMaid::init(number_settings)
+      dirty_array.each.with_index do |row, i|
+        new_row=[]
+        (new_row << clean_dates[i]).concat Array(NumberMaid::clean(row[1..-1]))     #add clean date and all clean numbers
+        clean_array << new_row
+      end
+
+      return Quandl::Babelfish::Data.new( clean_array, headers: header )
+    end
+  end
+end
+
+end
+end

data/lib/quandl/babelfish/data.rb

@@ -0,0 +1,22 @@
+module Quandl
+module Babelfish
+
+class Data < Array
+  
+  def initialize(*args, &block)
+    # do we have options?
+    options = args.pop if args && args.last.is_a?(Hash)
+    # set headers if given
+    @headers = options[:headers] if options && options.has_key?(:headers) && options[:headers].is_a?(Array)
+    # onwards and upwards
+    super(*args, &block)
+  end
+  
+  def headers
+    @headers ||= nil
+  end
+  
+end
+
+end
+end

data/lib/quandl/babelfish/date_maid.rb

@@ -0,0 +1,238 @@
+module Quandl
+module Babelfish
+
+  #responsible for number formatting
+  class DateMaid
+    @defaults = {
+        :format => nil
+    }
+
+    @settings = @defaults   #init with defaults
+
+    class << self
+
+      def init(user_settings)
+        @settings=@defaults.merge(user_settings)
+      end
+
+      #looks at all the dates and formats them to unambiguous ISO 8601 format  (yyyy-mm-dd)
+      def sweep(all_dates)
+        return nil if all_dates.nil?
+
+        all_dates = disinfect all_dates
+
+        if @settings[:format].nil?
+          #find good example and extract all info from it and apply it to each of the dates in the set
+          good_sample = find_good_date(all_dates)
+
+          raise Error::GuessDateFormat.new("Unable to find date format for provide dates") if good_sample.nil?
+
+          date_format, frequency = analyze_date_format(good_sample)
+
+
+        else
+          date_format = @settings[:format]
+        end
+
+        iso_dates=[]
+        all_dates.each do |fuzzy_date|
+          temp_date = convert(fuzzy_date, date_format) rescue raise(Error::InvalidDate,fuzzy_date)
+          iso_dates  << frequency_transform(temp_date, frequency)
+        end
+
+        iso_dates
+      end
+
+      def analyze_date_format(example)
+        return nil if example.nil?
+
+        # Regular formats and Custom formats (where Date.parse and Date.strptime
+        # fear to tread)
+        if re = example.match(/^(\d{1,2})\D(\d{1,2})\D\d{4}/) # eg "07/03/2012"
+          if re[1].to_i > 12
+            return '%d-%m-%Y', nil
+          else
+            return '%m-%d-%Y', nil
+          end
+        end
+        if re = example.match(/^(\d{1,2})\D(\d{1,2})\D\d{2}/) # eg "07/03/12"
+          if re[1].to_i > 12
+            return '%d-%m-%y', nil
+          else
+            return '%m-%d-%y', nil
+          end
+        end
+        # order these guys from most specific to most general
+        return "%Y", "annual" if example =~ /^\d{4}[\s]?-[\s]?\d{4}$/
+        return '%Y%m%d', 'daily' if example =~ /^\d{8}$/ && example[4..5].to_i < 13 && example[6..7].to_i < 32 # precisely 8 digits - yyyymmdd
+        return 'epoch', 'daily' if example =~ /^\d{7}.*$/ # 7 or more digits - epoch
+        return '%Y', 'annual' if example =~ /^\d{4}$/ # 4 digits
+        return '%Y', 'annual' if example =~ /^\d{4}\.0$/ # 4 digits with a dot 0 for excel
+        return ':year_quarter', 'quarterly' if example =~ /^\d{4}[Qq]\d$/ # 4 digits, Q, digit (here because the next pattern would override it)
+        return '%YM%m', 'monthly' if example =~ /^\d{4}M\d{1,2}$/ # 2007M08
+        return '%GW%V', 'weekly' if example =~ /^\d{4}W\d{1,2}$/ # 2012W01
+        return '%Y-%m', 'monthly' if example =~ /^\d{4}\D\d{1,2}$/ # 4 digits, separator, 1-2 digits
+        return '%m-%Y', 'monthly' if example =~ /^\d{1,2}\D\d{4}$/ # 1-2 digits, separator, 4 digits
+        return '%Y%m', 'monthly' if example =~ /^\d{6}$/ # 6 digits
+        return '%Y-%b', 'monthly' if example =~ /^\d{4}\D\w{3}$/ # 4 digits, separator, 3 letters
+        return '%b-%Y', 'monthly' if example =~ /^\w{3}\D\d{4}$/ # 3 letters, separator, 4 digits
+        return '%b-%y', 'monthly' if example =~ /^\w{3}\D\d{2}$/ # 3 letters, separator, 2 digits
+        return '%Y%b', 'monthly' if example =~ /^\d{4}\w{3}$/ # 4 digits, 3 letters
+        return '%b%Y', 'monthly' if example =~ /^\w{3}\d{4}$/ # 3 letters, 4 digits
+        return '%Y-%b-%d', 'daily' if example =~ /^\d{4}\D\w{3}\D\d{1,2}$/ # 4 digits, separator, 3 letters, separator, 1-2 digits
+        return '%Y-%m-%d', 'daily' if example =~ /^\d{4}\D\d{1,2}\D\d{1,2}$/ # 4 digits, separator, 1-2 digits, separator, 1-2 digits
+        return '%d-%b-%Y', 'daily' if example =~ /^\d{1,2}\D\w{3}\D\d{4}$/ # 1-2 digits, separator, 3 letters, separator, 4 digits
+        return '%Y%b%d', 'daily' if example =~ /^\d{4}\w{3}\d{1,2}$/ # 4 digits, 3 letters, 1-2 digits
+        return '%d%b%Y', 'daily' if example =~ /^\d{1,2}\w{3}\d{4}$/ # 1-2 digits, 3 letters, 4 digits
+        return '%d-%b-%y', 'daily' if example =~ /^\d{1,2}\D\w{3}\D\d{2}$/ # 1-2 digits, 3 letters, 2 digits
+        return '%b-%d-%Y', 'daily' if example =~ /^\w{3}\D\d{1,2}\D{1,2}\d{4}$/ # 3 letters, separator, 1-2 digits, separator(s), 4 digits
+
+        #our custom formats
+        return ':year_quarter', 'quarterly' if example =~ /^\d{4}\D[Qq]\d$/ # 4 digits, separator, Q, digit
+        return ':excel-1900', 'daily' if example =~ /^\d{5}$/ # 5 digits
+        return ':excel-1900', 'daily' if example =~ /^\d{5}\.0$/ # 5 digits dot zero excel
+
+        # No, try default date parse
+        # raise PostProcessorException, "Unable to guess date format for #{example}"
+        [nil, nil]
+      end
+
+      def disinfect(dates)
+        [*dates].collect do |date|
+          date.to_s.encode!('UTF-8', 'UTF-8', :invalid => :replace)
+          date.to_s.gsub!(/[^\x01-\x7f]/,'')
+          date.to_s.strip.gsub(/\s\s+/, ' ')
+        end
+      end
+      private
+
+
+      #converts date to specified format
+      def convert(fuzzy_date, date_format)
+        if date_format.nil?
+          # Assuming a US date format with 3 parameters (i.e. MM?DD?YYYY)
+          tokens = fuzzy_date.split(/\D/)
+          if tokens[0].length > 2 || fuzzy_date =~ /\w{2}/
+            # Its ISO
+            return DateTime.parse(fuzzy_date.to_s).to_date
+          else
+            # Guessing US
+            return Date.new(tokens[2].to_i, tokens[0].to_i, tokens[1].to_i)
+          end
+        else
+          case date_format
+          when ':year_quarter'
+            return year_quarter_formatter(fuzzy_date)
+          when ':excel-1900'
+            return excel_1900_formatter(fuzzy_date)
+          else #regular ruby formatter
+            return regular_formatter(fuzzy_date, date_format)
+          end
+
+        end
+      end
+
+
+      def year_quarter_formatter(fuzzy_date)
+        raw_date = fuzzy_date
+        tokens = raw_date.gsub(/[qQ]/, '-').gsub(/[a-zA-Z]/, '').split(/[^0-9]/)
+        tokens.delete_if {|x| x.nil? || x.empty?} # In case there are more than one delimiter because we replaced the Q
+        Date.new(tokens[0].to_i, tokens[1].to_i * 3, 1)
+      end
+
+      def excel_1900_formatter(fuzzy_date)
+        # handle Lotus 123 bug has 1900 as a leap year
+        Date.civil(1899, 12, 31) + fuzzy_date.to_i - 1 if fuzzy_date.to_i > 0
+      end
+
+      def regular_formatter(fuzzy_date, date_format)
+        # We have a date format - oh so pretty, but...
+        date_string = fuzzy_date
+        # normalize delimiters to hyphens so we do not have to make a format for each one.
+        # delimiters can be letters when its all numbers and delimiters only when there are letters. Sigh.
+        # only if no format where provided
+        date_string = date_string.gsub(/[^\d\w]+/, '-') if @settings[:format].nil?
+
+        #epoch date string
+        if date_format == 'epoch'
+          news = Time.at(date_string.to_i).utc.to_s.match(/\d\d\d\d-\d\d-\d\d/)
+          formatted_date = DateTime.strptime(news.to_s, '%Y-%m-%d').to_date
+        else
+          if date_string.to_s =~ /^(\w{3})\D(\d{2})$/
+            century = $2.to_i < 25 ? '20' : '19'
+            date_string = "#{$1} #{century}#{$2}"
+            formatted_date = DateTime.strptime(date_string.to_s, '%b %Y').to_date
+          else
+            formatted_date = DateTime.strptime(date_string.to_s, date_format).to_date
+          end
+        end
+        formatted_date+=4 if date_format == '%GW%V'  #strptime makes dates on Mondays. We want Fridays.
+        formatted_date
+      end
+
+
+
+      #find good example of date to use as template for format
+      def find_good_date(all_dates)
+        good_sample=nil
+        all_dates.each do |fuzzy_date|
+          if  usable_cell(fuzzy_date)
+            good_sample = fuzzy_date
+            break
+          end
+        end
+        good_sample
+      end
+
+      def usable_cell(cell)
+        return false if cell.nil? || cell.to_s.empty?
+        return false if cell.to_s.size > 20 # even annotated date can't be bigger than 20
+
+        return true if cell.to_s =~ /^\w{3}\D[456789]\d$/
+        # date is not usable as an example if it is ambiguous as to day and month
+        # 03/04/2012, for example, is ambiguous.  03/17/2012 is NOT ambiguous
+        if re = cell.to_s.match(/^(\d{1,2})\D(\d{1,2})\D\d{2,4}/)  # e.g. 03/04/2012
+          if re[1].to_i <= 12 and re[2].to_i <= 12
+            return false
+          else
+            return true
+          end
+        end
+
+        if re = cell.to_s.match(/^(\d{1,2})\D\w{3}\D(\d{2})/) # 07-jun-07
+          if re[1].to_i <= 12 and re[2].to_i <= 12
+            return false
+          else
+            return true
+          end
+        end
+
+        return true if cell.to_s =~ /\d{4}/ # It has a 4 digit year somewhere
+
+        return true if cell.to_s =~ /^\w{3}\D\d{2}/ # %b-%y(d)..also not ambiguous
+
+        false # Thank you, come again
+      end
+
+      # Bump date to the end of the respective periods
+      def frequency_transform(date, frequency)
+        case frequency
+          when 'annual'
+            date = Date.new(date.year,12,31)
+          when 'quarterly'
+            month = 3*((date.month-1)/3 + 1) # equals 3,6,9 or 12
+            date = Date.new(date.year, month, 1).next_month-1
+          when 'monthly'
+            date = Date.new(date.year, date.month,1).next_month-1
+          else
+            # Do nothing for daily or weekly
+        end
+
+        date
+      end
+
+
+    end
+  end
+end
+end

data/lib/quandl/babelfish/number_maid.rb

@@ -0,0 +1,80 @@
+module Quandl
+module Babelfish
+
+  #responsible for number cleaning
+  class NumberMaid
+    @defaults = {
+        :decimal_mark => Regexp.escape('.'),
+        :ignore_brackets => false, # Brackets ARE negative by default
+    }
+
+    @settings = @defaults   #init with defaults
+
+    class << self
+
+
+      def init(user_settings)
+        @settings=@defaults.merge(user_settings)
+        @escaped_decimal = Regexp.escape(@settings[:decimal_mark])
+      end
+
+      #cleans each number one by one
+      def clean(dirty_numbers)
+        return nil if dirty_numbers.nil?
+        numbers=[]
+        Array(dirty_numbers).each do |cell|
+          numbers << cell_to_number(cell.to_s)
+        end
+
+        (numbers.size == 1) ? numbers[0] : numbers
+      end
+
+      def cell_to_number(num)
+        return nil if num.nil?
+        # Remove annotations
+        # if there is something in parenthesis and a number elsewhere, nuke the parenthesis
+        temp = num.gsub(/[\(\[\{].*[\)\}\]]/, '')
+        num = temp if temp.match(/\d/)
+
+        num.gsub!("est.", '')
+
+        #check for exponents by searching for 'e' 'E' or variations of 'x 10' '*10' and 'X10^'
+        is_exp = false
+        expmultiplier = 1
+        m = /(\s)*(E|e|[X|x|\*](\s)*10(\^)?)(\s)*/.match(num)
+        #check if match is made, preceeded by a number/decimal, and succeeded by a digit or a plus/minus sign
+        if !m.nil? and m.pre_match =~ /[0-9#{@escaped_decimal}]$/ and m.post_match =~ /^([\-+0-9])/
+          is_exp = true
+          num = m.pre_match
+          expmultiplier = 10 ** /^[0-9\-+]*/.match(m.post_match)[0].to_i
+        end
+        is_million = (num =~ /million/i)
+        is_billion = (num =~ /billion/i)
+        is_negative = (num =~ /-[\d]/ or (!@settings[:ignore_brackets] and num =~ /\([\d]/))
+
+        # watch out for two numbers, like a range  eg "27.3 - 33.9"
+        # how: if you a see a number followed by a non number char that is not the decimal marker, kill everything to the right of that
+        num.gsub!(/(\d) (\d)/, '\1\2')
+        if m = num.match(/-?\s*[,\d\.]+/)
+          num = m[0]
+        end
+
+        # only keep #s and decimal mark
+        num.gsub!(/[^0-9#{@escaped_decimal}]/, '')
+        num.gsub!(/[^0-9]/, '.')
+
+        return nil if num.nil? || num !~ /[\d]/
+        return nil if num.end_with?(".")
+        return nil if num.count(".") > 1
+        cell = num.nil? ? 0.0 : Float("%.#{14}g" % num)
+        cell *= 1e6 if is_million
+        cell *= 1e9 if is_billion
+        cell *= -1 if is_negative
+        cell *= expmultiplier if is_exp
+        cell
+      end
+
+    end
+  end
+end
+end

data/lib/quandl/babelfish/version.rb

@@ -0,0 +1,5 @@
+module Quandl
+module Babelfish
+  VERSION = '0.0.2'
+end
+end

data/lib/quandl/babelfish.rb

@@ -0,0 +1,22 @@
+require "quandl/babelfish/version"
+
+require "quandl/babelfish/data"
+require "quandl/babelfish/cleaner"
+require "quandl/babelfish/date_maid"
+require "quandl/babelfish/number_maid"
+
+require 'quandl/error/guess_date_format'
+require 'quandl/error/invalid_date'
+require 'quandl/error/unknown_date_format'
+
+module Quandl
+module Babelfish
+
+  class << self
+    def clean(data, date_settings={}, number_settings={})
+      Cleaner::process data, date_settings, number_settings
+    end
+  end
+
+end
+end

data/lib/quandl/error/guess_date_format.rb

@@ -0,0 +1,6 @@
+module Quandl
+module Error
+  class GuessDateFormat < StandardError
+  end
+end
+end

data/lib/quandl/error/invalid_date.rb

@@ -0,0 +1,6 @@
+module Quandl
+module Error
+  class InvalidDate < StandardError
+  end
+end
+end

data/lib/quandl/error/unknown_date_format.rb

@@ -0,0 +1,6 @@
+module Quandl
+module Error
+  class UnknownDateFormat < StandardError
+  end
+end
+end

data/quandl_babelfish.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "quandl/babelfish/version"
+
+Gem::Specification.new do |s|
+  s.name        = "quandl_babelfish"
+  s.version     = Quandl::Babelfish::VERSION
+  s.authors     = ["Sergei Ryshkevich"]
+  s.email       = ["sergei@quandl.com"]
+  s.homepage    = "http://quandl.com/"
+  s.license     = "MIT"
+  s.summary     = "Quandl Data Cleaner"
+  s.description = "Quandl Data Cleaner"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rspec", "~> 2.13"
+  s.add_development_dependency "pry"
+end

data/spec/lib/quandl/babelfish/cleaner_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+include Quandl::Babelfish
+describe Cleaner do
+
+  let(:input){ [] }
+  let(:output){ Cleaner.process(input) }
+  subject{ output }
+  
+  context "garbage" do
+    let(:input){ [[2456624, 10], [2456625, 20], [2456626, 30]] }
+    it{ should be_eq_at_index '[0][0]', Date.new(1970,01,29) }
+  end
+  
+  context "headers with whitespace" do
+    let(:input){ [["   Date   ", " C1    ", "C2   ", "    C4"],[1990,1,2,3],[1991,4,5,6]] }
+    its(:headers){ should eq ["Date", "C1", "C2", "C4"] }
+  end
+  
+  context "annual" do
+    let(:input){ [[1990,1,2,3],[1991,4,5,6]] }
+    it{ should be_eq_at_index '[0][0]', Date.new(1990,12,31) }
+    it{ should be_eq_at_index '[0][1]', 1 }
+    it{ should be_eq_at_index '[1][0]', Date.new(1991,12,31) }
+    it{ should be_eq_at_index '[1][3]', 6 }
+    its(:headers){ should be_nil }
+  end
+  
+  context "numeric date" do
+    let(:input){ [[19900101,'1 [estimate]','2.3 - 4.0','not a number']] }
+    it{ should be_eq_at_index '[0][0]', Date.new(1990,01,01) }
+    it{ should be_eq_at_index '[0][1]', 1 }
+    it{ should be_eq_at_index '[0][2]', 2.3 }
+    it{ should be_eq_at_index '[0][3]', nil }
+    its(:headers){ should be_nil }
+  end
+  
+  context "data with headers" do
+    let(:input){ [['Date',0,0,0],[19900101,'1 [estimate]','2.3 - 4.0','not a number']] }
+    it{ should be_eq_at_index '[0][0]', Date.new(1990,01,01) }
+    it{ should be_eq_at_index '[0][1]', 1 }
+    it{ should be_eq_at_index '[0][2]', 2.3 }
+    it{ should be_eq_at_index '[0][3]', nil }
+    its(:headers){ should eq ['Date','0','0','0'] }
+  end
+
+end

data/spec/lib/quandl/babelfish/data_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe Quandl::Babelfish::Data do
+  let(:data_args){ [] }
+  subject{ Quandl::Babelfish::Data.new(*data_args) }
+  
+  its(:to_a){ should eq [] }
+  its(:headers){ should be_nil }
+    
+  context "given Array" do
+    let(:data_args){ [ [[1,2,3],[4,3,5]] ] }
+    its(:to_a){ should eq [[1,2,3],[4,3,5]] }
+    its(:headers){ should be_nil }
+  end
+  
+  context "given Array with :headers" do
+    let(:data_args){ [ [[1,2,3],[4,3,5]], { headers: ['Date', 'C1', 'C2'] } ] }
+    its(:to_a){ should eq [[1,2,3],[4,3,5]] }
+    its(:headers){ should eq ['Date', 'C1', 'C2'] }
+  end
+  
+  context "given junk headers: Float" do
+    let(:data_args){ [ 2, { headers: 1.2 } ] }
+    its(:to_a){ should eq [nil,nil] }
+    its(:headers){ should be_nil }
+  end
+  context "given junk headers: String" do
+    let(:data_args){ [ 2, { headers: '1.2' } ] }
+    its(:to_a){ should eq [nil,nil] }
+    its(:headers){ should be_nil }
+  end
+    
+end