quandl_babelfish 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +7 -7
- data/.travis.yml +12 -12
- data/Gemfile +1 -1
- data/LICENSE +7 -7
- data/README.md +18 -18
- data/UPGRADE.md +31 -29
- data/lib/quandl/babelfish/chronometer.rb +43 -43
- data/lib/quandl/babelfish/cleaner.rb +32 -32
- data/lib/quandl/babelfish/date_maid.rb +237 -237
- data/lib/quandl/babelfish/helper.rb +9 -0
- data/lib/quandl/babelfish/number_maid.rb +79 -79
- data/lib/quandl/babelfish/version.rb +4 -4
- data/lib/quandl/babelfish.rb +28 -27
- data/lib/quandl/error/guess_date_format.rb +4 -4
- data/lib/quandl/error/invalid_date.rb +4 -4
- data/lib/quandl/error/standard.rb +26 -26
- data/lib/quandl/error/unknown_date_format.rb +4 -4
- data/quandl_babelfish.gemspec +21 -21
- data/spec/lib/quandl/babelfish/chronometer_spec.rb +50 -50
- data/spec/lib/quandl/babelfish/cleaner_spec.rb +70 -49
- data/spec/lib/quandl/babelfish/date_maid_spec.rb +528 -528
- data/spec/lib/quandl/babelfish/helper_spec.rb +45 -0
- data/spec/lib/quandl/babelfish/number_maid_spec.rb +126 -126
- data/spec/lib/quandl/babelfish_spec.rb +15 -15
- data/spec/spec_helper.rb +12 -12
- data/spec/support/matchers/be_eq_at_index.rb +31 -31
- metadata +18 -17
- checksums.yaml +0 -7
data/lib/quandl/babelfish.rb
CHANGED
@@ -1,28 +1,29 @@
|
|
1
|
-
require "quandl/babelfish/version"
|
2
|
-
|
3
|
-
require "quandl/babelfish/
|
4
|
-
require "quandl/babelfish/
|
5
|
-
require "quandl/babelfish/
|
6
|
-
require "quandl/babelfish/
|
7
|
-
|
8
|
-
|
9
|
-
require 'quandl/error/
|
10
|
-
require 'quandl/error/
|
11
|
-
require 'quandl/error/
|
12
|
-
|
13
|
-
|
14
|
-
module
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
1
|
+
require "quandl/babelfish/version"
|
2
|
+
|
3
|
+
require "quandl/babelfish/helper"
|
4
|
+
require "quandl/babelfish/cleaner"
|
5
|
+
require "quandl/babelfish/date_maid"
|
6
|
+
require "quandl/babelfish/number_maid"
|
7
|
+
require "quandl/babelfish/chronometer"
|
8
|
+
|
9
|
+
require 'quandl/error/standard'
|
10
|
+
require 'quandl/error/guess_date_format'
|
11
|
+
require 'quandl/error/invalid_date'
|
12
|
+
require 'quandl/error/unknown_date_format'
|
13
|
+
|
14
|
+
module Quandl
|
15
|
+
module Babelfish
|
16
|
+
|
17
|
+
class << self
|
18
|
+
def clean(data, date_settings={}, number_settings={})
|
19
|
+
Cleaner::process data, date_settings, number_settings
|
20
|
+
end
|
21
|
+
|
22
|
+
def guess_frequency(data)
|
23
|
+
Chronometer::process data
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
28
29
|
end
|
@@ -1,5 +1,5 @@
|
|
1
|
-
module Quandl
|
2
|
-
module Error
|
3
|
-
class GuessDateFormat < Quandl::Error::Standard; end
|
4
|
-
end
|
1
|
+
module Quandl
|
2
|
+
module Error
|
3
|
+
class GuessDateFormat < Quandl::Error::Standard; end
|
4
|
+
end
|
5
5
|
end
|
@@ -1,5 +1,5 @@
|
|
1
|
-
module Quandl
|
2
|
-
module Error
|
3
|
-
class InvalidDate < Quandl::Error::Standard; end
|
4
|
-
end
|
1
|
+
module Quandl
|
2
|
+
module Error
|
3
|
+
class InvalidDate < Quandl::Error::Standard; end
|
4
|
+
end
|
5
5
|
end
|
@@ -1,27 +1,27 @@
|
|
1
|
-
module Quandl
|
2
|
-
module Error
|
3
|
-
class Standard < StandardError
|
4
|
-
|
5
|
-
attr_accessor :details
|
6
|
-
|
7
|
-
def line
|
8
|
-
detail :line
|
9
|
-
end
|
10
|
-
def context
|
11
|
-
detail :context
|
12
|
-
end
|
13
|
-
def problem
|
14
|
-
detail :problem
|
15
|
-
end
|
16
|
-
|
17
|
-
def detail(key)
|
18
|
-
details.send(key) if details.respond_to?(key)
|
19
|
-
end
|
20
|
-
|
21
|
-
def initialize(opts=nil)
|
22
|
-
@details = OpenStruct.new( opts ) if opts && opts.is_a?(Hash)
|
23
|
-
end
|
24
|
-
|
25
|
-
end
|
26
|
-
end
|
1
|
+
module Quandl
|
2
|
+
module Error
|
3
|
+
class Standard < StandardError
|
4
|
+
|
5
|
+
attr_accessor :details
|
6
|
+
|
7
|
+
def line
|
8
|
+
detail :line
|
9
|
+
end
|
10
|
+
def context
|
11
|
+
detail :context
|
12
|
+
end
|
13
|
+
def problem
|
14
|
+
detail :problem
|
15
|
+
end
|
16
|
+
|
17
|
+
def detail(key)
|
18
|
+
details.send(key) if details.respond_to?(key)
|
19
|
+
end
|
20
|
+
|
21
|
+
def initialize(opts=nil)
|
22
|
+
@details = OpenStruct.new( opts ) if opts && opts.is_a?(Hash)
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
27
|
end
|
@@ -1,5 +1,5 @@
|
|
1
|
-
module Quandl
|
2
|
-
module Error
|
3
|
-
class UnknownDateFormat < Quandl::Error::Standard; end
|
4
|
-
end
|
1
|
+
module Quandl
|
2
|
+
module Error
|
3
|
+
class UnknownDateFormat < Quandl::Error::Standard; end
|
4
|
+
end
|
5
5
|
end
|
data/quandl_babelfish.gemspec
CHANGED
@@ -1,21 +1,21 @@
|
|
1
|
-
# -*- encoding: utf-8 -*-
|
2
|
-
$:.push File.expand_path("../lib", __FILE__)
|
3
|
-
require "quandl/babelfish/version"
|
4
|
-
|
5
|
-
Gem::Specification.new do |s|
|
6
|
-
s.name = "quandl_babelfish"
|
7
|
-
s.version = Quandl::Babelfish::VERSION
|
8
|
-
s.authors = ["Sergei Ryshkevich"]
|
9
|
-
s.email = ["sergei@quandl.com"]
|
10
|
-
s.homepage = "http://quandl.com/"
|
11
|
-
s.license = "MIT"
|
12
|
-
s.summary = "Quandl Data Cleaner"
|
13
|
-
s.description = "Quandl Data Cleaner"
|
14
|
-
|
15
|
-
s.files = `git ls-files`.split("\n")
|
16
|
-
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
-
s.require_paths = ["lib"]
|
18
|
-
|
19
|
-
s.add_development_dependency "rspec", "~> 2.13"
|
20
|
-
s.add_development_dependency "pry"
|
21
|
-
end
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "quandl/babelfish/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "quandl_babelfish"
|
7
|
+
s.version = Quandl::Babelfish::VERSION
|
8
|
+
s.authors = ["Sergei Ryshkevich"]
|
9
|
+
s.email = ["sergei@quandl.com"]
|
10
|
+
s.homepage = "http://quandl.com/"
|
11
|
+
s.license = "MIT"
|
12
|
+
s.summary = "Quandl Data Cleaner"
|
13
|
+
s.description = "Quandl Data Cleaner"
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
s.require_paths = ["lib"]
|
18
|
+
|
19
|
+
s.add_development_dependency "rspec", "~> 2.13"
|
20
|
+
s.add_development_dependency "pry"
|
21
|
+
end
|
@@ -1,51 +1,51 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
include Quandl::Babelfish
|
4
|
-
describe Chronometer do
|
5
|
-
|
6
|
-
it 'should calculate frequency = daily' do
|
7
|
-
table = [['2012-01-01','1','2'],['2012-01-02','3','4'],['2012-01-03','5','6']]
|
8
|
-
frequency = Chronometer.process(table)
|
9
|
-
frequency.should == 'daily'
|
10
|
-
end
|
11
|
-
|
12
|
-
it 'should calculate frequency = monthly' do
|
13
|
-
table = [['2012-01-01','1','2'],['2012-02-01','3','4'],['2012-04-01','5','6'],
|
14
|
-
['2012-04-01','1','2'],['2012-05-01','3','4'],['2012-06-01','5','6']]
|
15
|
-
frequency = Chronometer.process(table)
|
16
|
-
frequency.should == 'monthly'
|
17
|
-
end
|
18
|
-
|
19
|
-
it 'should calculate frequency = quarterly' do
|
20
|
-
table = [['2012-01-01','1','2'],['2012-04-01','3','4'],['2012-07-01','5','6'],
|
21
|
-
['2012-10-01','1','2'],['2013-01-01','3','4'],['2012-04-01','5','6']]
|
22
|
-
frequency = Chronometer.process(table)
|
23
|
-
frequency.should == 'quarterly'
|
24
|
-
end
|
25
|
-
|
26
|
-
it 'should calculate frequency = quarterly' do
|
27
|
-
table = [['2012-01-01','1','2'],['2012-07-01','3','4'],['2013-01-01','5','6'],
|
28
|
-
['2013-07-01','1','2']]
|
29
|
-
frequency = Chronometer.process(table)
|
30
|
-
frequency.should == 'quarterly'
|
31
|
-
end
|
32
|
-
|
33
|
-
it 'should calculate frequency = annual' do
|
34
|
-
table = [['2008-01-01','1','2'],['2008-12-01','3','4'],['2010-01-01','5','6'],
|
35
|
-
['2011-01-01','1','2'],['2013-01-01','5','6']]
|
36
|
-
frequency = Chronometer.process(table)
|
37
|
-
frequency.should == 'annual'
|
38
|
-
end
|
39
|
-
|
40
|
-
it 'should calculate frequency = daily if only one row' do
|
41
|
-
table = [['2010-01-01','1','2']]
|
42
|
-
frequency = Chronometer.process(table)
|
43
|
-
frequency.should == 'daily'
|
44
|
-
end
|
45
|
-
|
46
|
-
it 'should calculate frequency = nil if nil table passed' do
|
47
|
-
frequency = Chronometer.process(nil)
|
48
|
-
frequency.should == nil
|
49
|
-
end
|
50
|
-
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
include Quandl::Babelfish
|
4
|
+
describe Chronometer do
|
5
|
+
|
6
|
+
it 'should calculate frequency = daily' do
|
7
|
+
table = [['2012-01-01','1','2'],['2012-01-02','3','4'],['2012-01-03','5','6']]
|
8
|
+
frequency = Chronometer.process(table)
|
9
|
+
frequency.should == 'daily'
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'should calculate frequency = monthly' do
|
13
|
+
table = [['2012-01-01','1','2'],['2012-02-01','3','4'],['2012-04-01','5','6'],
|
14
|
+
['2012-04-01','1','2'],['2012-05-01','3','4'],['2012-06-01','5','6']]
|
15
|
+
frequency = Chronometer.process(table)
|
16
|
+
frequency.should == 'monthly'
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'should calculate frequency = quarterly' do
|
20
|
+
table = [['2012-01-01','1','2'],['2012-04-01','3','4'],['2012-07-01','5','6'],
|
21
|
+
['2012-10-01','1','2'],['2013-01-01','3','4'],['2012-04-01','5','6']]
|
22
|
+
frequency = Chronometer.process(table)
|
23
|
+
frequency.should == 'quarterly'
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'should calculate frequency = quarterly' do
|
27
|
+
table = [['2012-01-01','1','2'],['2012-07-01','3','4'],['2013-01-01','5','6'],
|
28
|
+
['2013-07-01','1','2']]
|
29
|
+
frequency = Chronometer.process(table)
|
30
|
+
frequency.should == 'quarterly'
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'should calculate frequency = annual' do
|
34
|
+
table = [['2008-01-01','1','2'],['2008-12-01','3','4'],['2010-01-01','5','6'],
|
35
|
+
['2011-01-01','1','2'],['2013-01-01','5','6']]
|
36
|
+
frequency = Chronometer.process(table)
|
37
|
+
frequency.should == 'annual'
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'should calculate frequency = daily if only one row' do
|
41
|
+
table = [['2010-01-01','1','2']]
|
42
|
+
frequency = Chronometer.process(table)
|
43
|
+
frequency.should == 'daily'
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'should calculate frequency = nil if nil table passed' do
|
47
|
+
frequency = Chronometer.process(nil)
|
48
|
+
frequency.should == nil
|
49
|
+
end
|
50
|
+
|
51
51
|
end
|
@@ -1,49 +1,70 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
include Quandl::Babelfish
|
4
|
-
describe Cleaner do
|
5
|
-
|
6
|
-
let(:input){ [] }
|
7
|
-
let(:output){ Cleaner.process(input) }
|
8
|
-
let(:data){ output[0] }
|
9
|
-
let(:headers){ output[1] }
|
10
|
-
subject{ data }
|
11
|
-
|
12
|
-
context "garbage" do
|
13
|
-
let(:input){ [[2456624, 10], [2456625, 20], [2456626, 30]] }
|
14
|
-
it{ should be_eq_at_index '[0][0]', Date.new(1970,01,29) }
|
15
|
-
end
|
16
|
-
|
17
|
-
context "headers with whitespace" do
|
18
|
-
let(:input){ [[" Date ", " C1 ", "C2 ", " C4"],[1990,1,2,3],[1991,4,5,6]] }
|
19
|
-
it{ headers.should eq ["Date", "C1", "C2", "C4"] }
|
20
|
-
end
|
21
|
-
|
22
|
-
context "annual" do
|
23
|
-
let(:input){ [[1990,1,2,3],[1991,4,5,6]] }
|
24
|
-
it{ should be_eq_at_index '[0][0]', Date.new(1990,12,31) }
|
25
|
-
it{ should be_eq_at_index '[0][1]', 1 }
|
26
|
-
it{ should be_eq_at_index '[1][0]', Date.new(1991,12,31) }
|
27
|
-
it{ should be_eq_at_index '[1][3]', 6 }
|
28
|
-
it{ headers.should be_nil }
|
29
|
-
end
|
30
|
-
|
31
|
-
context "numeric date" do
|
32
|
-
let(:input){ [[19900101,'1 [estimate]','2.3 - 4.0','not a number']] }
|
33
|
-
it{ should be_eq_at_index '[0][0]', Date.new(1990,01,01) }
|
34
|
-
it{ should be_eq_at_index '[0][1]', 1 }
|
35
|
-
it{ should be_eq_at_index '[0][2]', 2.3 }
|
36
|
-
it{ should be_eq_at_index '[0][3]', nil }
|
37
|
-
it{ headers.should be_nil }
|
38
|
-
end
|
39
|
-
|
40
|
-
context "data with headers" do
|
41
|
-
let(:input){ [['Date',0,0,0],[19900101,'1 [estimate]','2.3 - 4.0','not a number']] }
|
42
|
-
it{ should be_eq_at_index '[0][0]', Date.new(1990,01,01) }
|
43
|
-
it{ should be_eq_at_index '[0][1]', 1 }
|
44
|
-
it{ should be_eq_at_index '[0][2]', 2.3 }
|
45
|
-
it{ should be_eq_at_index '[0][3]', nil }
|
46
|
-
it{ headers.should eq ['Date','0','0','0'] }
|
47
|
-
end
|
48
|
-
|
49
|
-
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
include Quandl::Babelfish
|
4
|
+
describe Cleaner do
|
5
|
+
|
6
|
+
let(:input){ [] }
|
7
|
+
let(:output){ Cleaner.process(input) }
|
8
|
+
let(:data){ output[0] }
|
9
|
+
let(:headers){ output[1] }
|
10
|
+
subject{ data }
|
11
|
+
|
12
|
+
context "garbage" do
|
13
|
+
let(:input){ [[2456624, 10], [2456625, 20], [2456626, 30]] }
|
14
|
+
it{ should be_eq_at_index '[0][0]', Date.new(1970,01,29) }
|
15
|
+
end
|
16
|
+
|
17
|
+
context "headers with whitespace" do
|
18
|
+
let(:input){ [[" Date ", " C1 ", "C2 ", " C4"],[1990,1,2,3],[1991,4,5,6]] }
|
19
|
+
it{ headers.should eq ["Date", "C1", "C2", "C4"] }
|
20
|
+
end
|
21
|
+
|
22
|
+
context "annual" do
|
23
|
+
let(:input){ [[1990,1,2,3],[1991,4,5,6]] }
|
24
|
+
it{ should be_eq_at_index '[0][0]', Date.new(1990,12,31) }
|
25
|
+
it{ should be_eq_at_index '[0][1]', 1 }
|
26
|
+
it{ should be_eq_at_index '[1][0]', Date.new(1991,12,31) }
|
27
|
+
it{ should be_eq_at_index '[1][3]', 6 }
|
28
|
+
it{ headers.should be_nil }
|
29
|
+
end
|
30
|
+
|
31
|
+
context "numeric date" do
|
32
|
+
let(:input){ [[19900101,'1 [estimate]','2.3 - 4.0','not a number']] }
|
33
|
+
it{ should be_eq_at_index '[0][0]', Date.new(1990,01,01) }
|
34
|
+
it{ should be_eq_at_index '[0][1]', 1 }
|
35
|
+
it{ should be_eq_at_index '[0][2]', 2.3 }
|
36
|
+
it{ should be_eq_at_index '[0][3]', nil }
|
37
|
+
it{ headers.should be_nil }
|
38
|
+
end
|
39
|
+
|
40
|
+
context "data with headers" do
|
41
|
+
let(:input){ [['Date',0,0,0],[19900101,'1 [estimate]','2.3 - 4.0','not a number']] }
|
42
|
+
it{ should be_eq_at_index '[0][0]', Date.new(1990,01,01) }
|
43
|
+
it{ should be_eq_at_index '[0][1]', 1 }
|
44
|
+
it{ should be_eq_at_index '[0][2]', 2.3 }
|
45
|
+
it{ should be_eq_at_index '[0][3]', nil }
|
46
|
+
it{ headers.should eq ['Date','0','0','0'] }
|
47
|
+
end
|
48
|
+
|
49
|
+
context "data with nil" do
|
50
|
+
let(:input){ [["Date", "Col1"], ["2002", nil], ["2003", "5"]] }
|
51
|
+
it{ should be_eq_at_index '[0][0]', Date.new(2002,12,31) }
|
52
|
+
it{ data[0].length.should ==2}
|
53
|
+
it{ should be_eq_at_index '[0][1]', nil }
|
54
|
+
it{ should be_eq_at_index '[1][0]', Date.new(2003,12,31) }
|
55
|
+
it{ should be_eq_at_index '[1][1]', 5 }
|
56
|
+
it{ headers.should eq ['Date','Col1'] }
|
57
|
+
end
|
58
|
+
|
59
|
+
context "data with middle nil" do
|
60
|
+
let(:input){ [["Date", "Col1", "Col2"], ["2002", nil, '1'], ["2003", "5", '6']] }
|
61
|
+
it{ should be_eq_at_index '[0][0]', Date.new(2002,12,31) }
|
62
|
+
it{ should be_eq_at_index '[0][1]', nil }
|
63
|
+
it{ should be_eq_at_index '[0][2]', 1}
|
64
|
+
it{ should be_eq_at_index '[1][0]', Date.new(2003,12,31) }
|
65
|
+
it{ should be_eq_at_index '[1][1]', 5 }
|
66
|
+
it{ should be_eq_at_index '[1][2]', 6 }
|
67
|
+
it{ headers.should eq ['Date','Col1', 'Col2'] }
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|