quandl_operation 0.4.1 → 0.4.2.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rubocop.yml +19 -0
- data/Gemfile +1 -1
- data/Guardfile +3 -4
- data/Rakefile +8 -8
- data/VERSION +1 -1
- data/lib/quandl/operation.rb +6 -6
- data/lib/quandl/operation/collapse.rb +120 -116
- data/lib/quandl/operation/collapse/guess.rb +77 -83
- data/lib/quandl/operation/core_ext.rb +1 -1
- data/lib/quandl/operation/core_ext/array.rb +16 -14
- data/lib/quandl/operation/core_ext/date.rb +22 -24
- data/lib/quandl/operation/core_ext/float.rb +1 -3
- data/lib/quandl/operation/core_ext/string.rb +6 -4
- data/lib/quandl/operation/core_ext/time.rb +20 -22
- data/lib/quandl/operation/qdate.rb +14 -18
- data/lib/quandl/operation/sort.rb +24 -28
- data/lib/quandl/operation/transform.rb +145 -151
- data/lib/quandl/operation/value.rb +15 -17
- data/lib/quandl/operation/version.rb +3 -3
- data/quandl_operation.gemspec +25 -25
- data/spec/lib/quandl/operation/collapse_spec.rb +84 -77
- data/spec/lib/quandl/operation/date_spec.rb +15 -20
- data/spec/lib/quandl/operation/transform_spec.rb +15 -19
- data/spec/spec_helper.rb +4 -4
- metadata +50 -64
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 72987e8e8aaab8342e2311e9205cb7a3a933510e
|
4
|
+
data.tar.gz: b212ef085932a197ba784ef3e06a011bf4c4b49e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 415c5c15bd5232bb50866dcd9287fe2476f5d586e743e52d328c688cfd94b80c879019f840063f3c92a88dd89a8dfadc7885a44d7fabca1b5275b22b1f8694a6
|
7
|
+
data.tar.gz: f5a428b56387e30ba758eb0e28a8dc59dd967ef3141f7499d38a5deea63e1a2d76fb459d2d5d8c410b2b1658d45df5308f8e5b9ca8d5c7f5c5010dce2a9dcca3
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
AllCops:
|
2
|
+
DisplayCopNames: true
|
3
|
+
DisplayStyleGuide: true
|
4
|
+
LineLength:
|
5
|
+
Max: 120
|
6
|
+
Style/Documentation:
|
7
|
+
Enabled: false
|
8
|
+
Metrics/MethodLength:
|
9
|
+
Max: 40
|
10
|
+
Metrics/PerceivedComplexity:
|
11
|
+
Enabled: false
|
12
|
+
Metrics/CyclomaticComplexity:
|
13
|
+
Enabled: false
|
14
|
+
Metrics/AbcSize:
|
15
|
+
Enabled: false
|
16
|
+
Metrics/MethodLength:
|
17
|
+
Max: 100
|
18
|
+
Metrics/ClassLength:
|
19
|
+
Max: 180
|
data/Gemfile
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
source
|
1
|
+
source 'https://rubygems.org'
|
2
2
|
gemspec
|
data/Guardfile
CHANGED
@@ -1,8 +1,7 @@
|
|
1
1
|
group :specs do
|
2
2
|
guard :rspec, cmd: 'bundle exec rspec --fail-fast -f doc --color' do
|
3
|
-
|
4
|
-
|
5
|
-
|
3
|
+
watch(%r{^spec/.+_spec\.rb$})
|
4
|
+
watch(%r{^(lib/.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
|
5
|
+
watch('spec/spec_helper.rb') { 'spec' }
|
6
6
|
end
|
7
7
|
end
|
8
|
-
|
data/Rakefile
CHANGED
@@ -1,13 +1,13 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
1
|
+
require 'bundler'
|
2
|
+
require 'rake'
|
3
|
+
require 'bundler/gem_tasks'
|
4
|
+
require 'rspec/core/rake_task'
|
5
5
|
|
6
|
-
task :
|
6
|
+
task default: :spec
|
7
7
|
|
8
|
-
desc
|
8
|
+
desc 'Run all specs'
|
9
9
|
RSpec::Core::RakeTask.new(:spec) do |task|
|
10
|
-
task.pattern =
|
10
|
+
task.pattern = 'spec/**/*_spec.rb'
|
11
11
|
end
|
12
12
|
|
13
13
|
require 'quandl/utility/rake_tasks'
|
@@ -16,5 +16,5 @@ Quandl::Utility::Tasks.configure do |c|
|
|
16
16
|
c.version_path = 'VERSION'
|
17
17
|
c.changelog_path = 'UPGRADE.md'
|
18
18
|
c.tag_prefix = 'v'
|
19
|
-
c.changelog_matching = ['^QUGC','^WIKI']
|
19
|
+
c.changelog_matching = ['^QUGC', '^WIKI']
|
20
20
|
end
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.4.
|
1
|
+
0.4.2.rc1
|
data/lib/quandl/operation.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
require 'quandl/logger'
|
2
2
|
|
3
|
-
require
|
3
|
+
require 'quandl/operation/version'
|
4
4
|
|
5
5
|
require 'csv'
|
6
6
|
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
7
|
+
require 'active_support'
|
8
|
+
require 'active_support/inflector'
|
9
|
+
require 'active_support/core_ext/hash'
|
10
10
|
|
11
11
|
require 'quandl/operation/core_ext'
|
12
12
|
require 'quandl/operation/collapse'
|
@@ -16,6 +16,6 @@ require 'quandl/operation/qdate'
|
|
16
16
|
require 'quandl/operation/value'
|
17
17
|
|
18
18
|
module Quandl
|
19
|
-
module Operation
|
19
|
+
module Operation
|
20
|
+
end
|
20
21
|
end
|
21
|
-
end
|
@@ -2,126 +2,130 @@
|
|
2
2
|
require 'quandl/operation/collapse/guess'
|
3
3
|
# collapse
|
4
4
|
module Quandl
|
5
|
-
module Operation
|
6
|
-
|
7
|
-
class
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
data = Sort.desc(data) if order == :desc
|
23
|
-
# onwards
|
24
|
-
data
|
25
|
-
end
|
26
|
-
|
27
|
-
def assert_valid_arguments!(data, type)
|
28
|
-
raise ArgumentError, "data must be an Array. Received: #{data.class}" unless data.is_a?(Array)
|
29
|
-
raise ArgumentError, "frequency must be one of #{valid_collapses}. Received: #{type}" unless valid?(type)
|
30
|
-
end
|
31
|
-
|
32
|
-
def valid_collapse?(type)
|
33
|
-
valid?(type)
|
34
|
-
end
|
35
|
-
|
36
|
-
def valid?(type)
|
37
|
-
valid_collapses.include?( type.try(:to_sym) )
|
38
|
-
end
|
39
|
-
|
40
|
-
def valid_collapses
|
41
|
-
[ :daily, :weekly, :monthly, :quarterly, :annual ]
|
42
|
-
end
|
43
|
-
|
44
|
-
def collapse(data, frequency)
|
45
|
-
return data unless valid_collapse?( frequency )
|
46
|
-
# store the new collapsed data
|
47
|
-
collapsed_data = {}
|
48
|
-
range = find_end_of_range( data[0][0], frequency )
|
49
|
-
# iterate over the data
|
50
|
-
data.each do |row|
|
51
|
-
# grab date and values
|
52
|
-
date, value = row[0], row[1..-1]
|
53
|
-
value = value.first if value.count == 1
|
54
|
-
# bump to the next range if it exceeds the current one
|
55
|
-
range = find_end_of_range(date, frequency) unless inside_range?(date, range)
|
56
|
-
# consider the value for the next range
|
57
|
-
if inside_range?(date, range) && value.present?
|
58
|
-
# merge this and previous row if nils are present
|
59
|
-
value = merge_row_values( value, collapsed_data[range] ) unless collapsed_data[range].nil?
|
60
|
-
# assign value
|
61
|
-
collapsed_data[range] = value
|
5
|
+
module Operation
|
6
|
+
class Collapse
|
7
|
+
class << self
|
8
|
+
def perform(data, type)
|
9
|
+
assert_valid_arguments!(data, type)
|
10
|
+
# nothing to do with an empty array
|
11
|
+
return data unless data.compact.present?
|
12
|
+
# source order
|
13
|
+
order = Sort.order?(data)
|
14
|
+
# operations expect data in ascending order
|
15
|
+
data = Sort.asc(data)
|
16
|
+
# collapse
|
17
|
+
data = collapse(data, type)
|
18
|
+
# return to original order
|
19
|
+
data = Sort.desc(data) if order == :desc
|
20
|
+
# onwards
|
21
|
+
data
|
62
22
|
end
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
def to_table(data)
|
68
|
-
data.collect do |date, values|
|
69
|
-
if values.is_a?(Array)
|
70
|
-
values.unshift(date)
|
71
|
-
else
|
72
|
-
[date, values]
|
23
|
+
|
24
|
+
def assert_valid_arguments!(data, type)
|
25
|
+
fail ArgumentError, "data must be an Array. Received: #{data.class}" unless data.is_a?(Array)
|
26
|
+
fail ArgumentError, "frequency must be one of #{valid_collapses}. Received: #{type}" unless valid?(type)
|
73
27
|
end
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
28
|
+
|
29
|
+
def valid_collapse?(type)
|
30
|
+
valid?(type)
|
31
|
+
end
|
32
|
+
|
33
|
+
def valid?(type)
|
34
|
+
valid_collapses.include?(type.try(:to_sym))
|
35
|
+
end
|
36
|
+
|
37
|
+
def valid_collapses
|
38
|
+
[:daily, :weekly, :monthly, :quarterly, :annual]
|
39
|
+
end
|
40
|
+
|
41
|
+
def collapse(data, frequency)
|
42
|
+
return data unless valid_collapse?(frequency)
|
43
|
+
|
44
|
+
# Special scenario where we are only fetching the `date` column
|
45
|
+
date_column_only = data.count > 0 && data[0].count == 1
|
46
|
+
|
47
|
+
# store the new collapsed data
|
48
|
+
collapsed_data = {}
|
49
|
+
range = find_end_of_range(data[0][0], frequency)
|
50
|
+
|
51
|
+
# iterate over the data
|
52
|
+
data.each do |row|
|
53
|
+
# grab date and values
|
54
|
+
date = row[0]
|
55
|
+
value = row[1..-1]
|
56
|
+
value = value.first if value.count == 1
|
57
|
+
|
58
|
+
# bump to the next range if it exceeds the current one
|
59
|
+
range = find_end_of_range(date, frequency) unless inside_range?(date, range)
|
60
|
+
|
61
|
+
# consider the value for the next range
|
62
|
+
next unless inside_range?(date, range) && (value.present? || date_column_only)
|
63
|
+
value = merge_row_values(value, collapsed_data[range]) unless collapsed_data[range].nil?
|
64
|
+
# assign value
|
65
|
+
collapsed_data[range] = value
|
66
|
+
end
|
67
|
+
|
68
|
+
to_table(collapsed_data)
|
69
|
+
end
|
70
|
+
|
71
|
+
def to_table(data)
|
72
|
+
data.collect do |date, values|
|
73
|
+
if values.is_a?(Array)
|
74
|
+
values.unshift(date)
|
75
|
+
else
|
76
|
+
[date, values]
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def merge_row_values(top_row, bottom_row)
|
82
|
+
# merge previous values when nils are present
|
83
|
+
if top_row.is_a?(Array) && top_row.include?(nil)
|
84
|
+
# find nil indexes
|
85
|
+
indexes = find_each_index(top_row, nil)
|
86
|
+
# merge nils with previous values
|
87
|
+
indexes.each { |index| top_row[index] = bottom_row[index] }
|
88
|
+
end
|
89
|
+
top_row
|
90
|
+
end
|
91
|
+
|
92
|
+
def collapses_greater_than(freq)
|
93
|
+
return [] unless freq.respond_to?(:to_sym)
|
94
|
+
index = valid_collapses.index(freq.to_sym)
|
95
|
+
index.present? ? valid_collapses.slice(index + 1, valid_collapses.count) : []
|
96
|
+
end
|
97
|
+
|
98
|
+
def collapses_greater_than_or_equal_to(freq)
|
99
|
+
return [] unless freq.respond_to?(:to_sym)
|
100
|
+
valid_collapses.slice(valid_collapses.index(freq.to_sym), valid_collapses.count)
|
101
|
+
end
|
102
|
+
|
103
|
+
def frequency?(data)
|
104
|
+
Guess.frequency(data)
|
105
|
+
end
|
106
|
+
|
107
|
+
def inside_range?(date, range)
|
108
|
+
date <= range
|
109
|
+
end
|
110
|
+
|
111
|
+
def find_end_of_range(date, frequency)
|
112
|
+
date.end_of_frequency(frequency)
|
113
|
+
end
|
114
|
+
|
115
|
+
def find_each_index(array, find)
|
116
|
+
found = -1
|
117
|
+
index = -1
|
118
|
+
q = []
|
119
|
+
while found
|
120
|
+
found = array[index + 1..-1].index(find)
|
121
|
+
if found
|
122
|
+
index = index + found + 1
|
123
|
+
q << index
|
124
|
+
end
|
125
|
+
end
|
126
|
+
q
|
118
127
|
end
|
119
128
|
end
|
120
|
-
q
|
121
129
|
end
|
122
|
-
|
123
130
|
end
|
124
|
-
|
125
|
-
end
|
126
131
|
end
|
127
|
-
end
|
@@ -1,90 +1,84 @@
|
|
1
1
|
module Quandl
|
2
|
-
module Operation
|
3
|
-
class Collapse
|
2
|
+
module Operation
|
3
|
+
class Collapse
|
4
|
+
class Guess
|
5
|
+
class << self
|
6
|
+
def frequency(data)
|
7
|
+
return :annual unless data && data[0] && data[0][0]
|
8
|
+
# find the smallest point of difference between dates
|
9
|
+
gap = find_average_gap(data)
|
10
|
+
# ensure gap is not negative
|
11
|
+
gap = ensure_positive_gap(gap)
|
12
|
+
# determine the freq from the size of the smallest gap
|
13
|
+
frequency_from_gap(gap)
|
14
|
+
end
|
4
15
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
date = row[0]
|
28
|
-
# only if pdate is present
|
29
|
-
if pdate
|
30
|
-
# calculate the gap
|
31
|
-
diff = (pdate - date).to_i
|
32
|
-
# replace the previous gap if it is smaller
|
33
|
-
gap = diff if diff < gap
|
34
|
-
end
|
35
|
-
# previous row's date
|
36
|
-
pdate = date
|
37
|
-
end
|
38
|
-
gap
|
39
|
-
end
|
40
|
-
|
41
|
-
def find_average_gap(data)
|
42
|
-
# init
|
43
|
-
gap = 100_000
|
44
|
-
pdate = nil
|
45
|
-
row_count = data.count
|
46
|
-
majority_count = (row_count * 0.55).to_i
|
47
|
-
gaps = {}
|
48
|
-
# find the smallest gap
|
49
|
-
data.each do |row|
|
50
|
-
# this row's date
|
51
|
-
date = row[0]
|
52
|
-
# only if pdate is present
|
53
|
-
if pdate
|
54
|
-
# calculate the gap
|
55
|
-
diff = (pdate - date).to_i
|
56
|
-
# increment the gap counter
|
57
|
-
gaps[diff] ||= 0
|
58
|
-
gaps[diff] += 1
|
59
|
-
# if the diff count is greater than majority_count, we have a consensus
|
60
|
-
return diff if gaps[diff] > majority_count
|
61
|
-
end
|
62
|
-
# previous row's date
|
63
|
-
pdate = date
|
64
|
-
end
|
65
|
-
gaps.to_a.sort_by{|r| r[1] }.try(:last).try(:first)
|
66
|
-
end
|
16
|
+
def find_smallest_gap(data)
|
17
|
+
# init
|
18
|
+
gap = 100_000
|
19
|
+
pdate = nil
|
20
|
+
# find the smallest gap
|
21
|
+
data.each do |row|
|
22
|
+
# if the gap is 1, we're done
|
23
|
+
break if gap <= 1
|
24
|
+
# this row's date
|
25
|
+
date = row[0]
|
26
|
+
# only if pdate is present
|
27
|
+
if pdate
|
28
|
+
# calculate the gap
|
29
|
+
diff = (pdate - date).to_i
|
30
|
+
# replace the previous gap if it is smaller
|
31
|
+
gap = diff if diff < gap
|
32
|
+
end
|
33
|
+
# previous row's date
|
34
|
+
pdate = date
|
35
|
+
end
|
36
|
+
gap
|
37
|
+
end
|
67
38
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
39
|
+
def find_average_gap(data)
|
40
|
+
# init
|
41
|
+
pdate = nil
|
42
|
+
row_count = data.count
|
43
|
+
majority_count = (row_count * 0.55).to_i
|
44
|
+
gaps = {}
|
45
|
+
# find the smallest gap
|
46
|
+
data.each do |row|
|
47
|
+
# this row's date
|
48
|
+
date = row[0]
|
49
|
+
# only if pdate is present
|
50
|
+
if pdate
|
51
|
+
# calculate the gap
|
52
|
+
diff = (pdate - date).to_i
|
53
|
+
# increment the gap counter
|
54
|
+
gaps[diff] ||= 0
|
55
|
+
gaps[diff] += 1
|
56
|
+
# if the diff count is greater than majority_count, we have a consensus
|
57
|
+
return diff if gaps[diff] > majority_count
|
58
|
+
end
|
59
|
+
# previous row's date
|
60
|
+
pdate = date
|
61
|
+
end
|
62
|
+
gaps.to_a.sort_by { |r| r[1] }.try(:last).try(:first)
|
63
|
+
end
|
64
|
+
|
65
|
+
def frequency_from_gap(gap)
|
66
|
+
case
|
67
|
+
when gap <= 1 then :daily
|
68
|
+
when gap <= 10 then :weekly
|
69
|
+
when gap <= 31 then :monthly
|
70
|
+
when gap <= 93 then :quarterly
|
71
|
+
else :annual
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def ensure_positive_gap(gap)
|
76
|
+
gap = gap.to_i
|
77
|
+
gap *= -1 if gap < 0
|
78
|
+
gap
|
79
|
+
end
|
80
|
+
end
|
76
81
|
end
|
77
82
|
end
|
78
|
-
|
79
|
-
def ensure_positive_gap(gap)
|
80
|
-
gap = gap.to_i
|
81
|
-
gap = gap * -1 if gap < 0
|
82
|
-
gap
|
83
|
-
end
|
84
|
-
|
85
83
|
end
|
86
|
-
|
87
|
-
end
|
88
|
-
end
|
89
84
|
end
|
90
|
-
end
|