embulk-input-mixpanel 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/embulk-input-mixpanel.gemspec +1 -1
- data/lib/embulk/input/mixpanel_api/client.rb +17 -8
- data/test/embulk/input/mixpanel_api/test_client.rb +30 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3c39e9af49ffa9cfcc15fe3156896652720fb72e
|
4
|
+
data.tar.gz: 350093593fbd3f62c80a10071cdcb473f36203e3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a7fe90abd0541686bb1dc4e5426a65524d93d02d4781d3f96b30d2cc8db97231302df64a11db378279a665bafeb086712d0bc3f465588f2e0bb0d9003e8cd34f
|
7
|
+
data.tar.gz: 89953f9317aae738fc331f878dde1096cfa7e53f8ef869081957306cf9fec5097f10d7affe9a19e467a5f22bf91b8af72b087aa136f550f55ed1e93ff4f7bcf9
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 0.4.5 - 2016-09-05
|
2
|
+
* [fixed] Don't try to guess future date [#43](https://github.com/treasure-data/embulk-input-mixpanel/pull/43)
|
3
|
+
|
1
4
|
## 0.4.4 - 2016-09-02
|
2
5
|
* [enhancement] Reduce memory usage by streaming processing [#42](https://github.com/treasure-data/embulk-input-mixpanel/pull/42)
|
3
6
|
|
@@ -56,13 +56,11 @@ module Embulk
|
|
56
56
|
end
|
57
57
|
|
58
58
|
def export_for_small_dataset(params = {})
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
try_to_dates.each do |to_date|
|
59
|
+
yesterday = Date.today - 1
|
60
|
+
latest_tried_to_date = nil
|
61
|
+
try_to_dates(params["from_date"]).each do |to_date|
|
62
|
+
next if yesterday < to_date
|
63
|
+
latest_tried_to_date = to_date
|
66
64
|
params["to_date"] = to_date.strftime("%Y-%m-%d")
|
67
65
|
records = retryer.with_retry do
|
68
66
|
request_small_dataset(params, SMALLSET_BYTE_RANGE)
|
@@ -71,7 +69,18 @@ module Embulk
|
|
71
69
|
return records
|
72
70
|
end
|
73
71
|
|
74
|
-
raise ConfigError.new "#{params["from_date"]}..#{
|
72
|
+
raise ConfigError.new "#{params["from_date"]}..#{latest_tried_to_date} has no record."
|
73
|
+
end
|
74
|
+
|
75
|
+
def try_to_dates(from_date)
|
76
|
+
try_to_dates = 5.times.map do |n|
|
77
|
+
# from_date + 1, from_date + 10, from_date + 100, ... so on
|
78
|
+
days = 1 * (10 ** n)
|
79
|
+
Date.parse(from_date.to_s) + days
|
80
|
+
end
|
81
|
+
yesterday = Date.today - 1
|
82
|
+
try_to_dates << yesterday
|
83
|
+
try_to_dates.find_all {|date| date <= yesterday}.uniq
|
75
84
|
end
|
76
85
|
|
77
86
|
private
|
@@ -30,6 +30,36 @@ module Embulk
|
|
30
30
|
assert_equal(expected, @client.__send__(:signature, params))
|
31
31
|
end
|
32
32
|
|
33
|
+
class TryToDatesTest < self
|
34
|
+
def setup
|
35
|
+
@client = Client.new(API_KEY, API_SECRET)
|
36
|
+
end
|
37
|
+
|
38
|
+
data do
|
39
|
+
[
|
40
|
+
["2000-01-01", "2000-01-01"],
|
41
|
+
["2010-01-01", "2010-01-01"],
|
42
|
+
["3 days ago", (Date.today - 3).to_s],
|
43
|
+
]
|
44
|
+
end
|
45
|
+
def test_candidates(from_str)
|
46
|
+
from = Date.parse(from_str)
|
47
|
+
yesterday = Date.today - 1
|
48
|
+
dates = @client.try_to_dates(from.to_s)
|
49
|
+
expect = [
|
50
|
+
from + 1,
|
51
|
+
from + 10,
|
52
|
+
from + 100,
|
53
|
+
from + 1000,
|
54
|
+
from + 10000,
|
55
|
+
yesterday,
|
56
|
+
].find_all{|d| d <= yesterday}
|
57
|
+
|
58
|
+
assert_equal expect, dates
|
59
|
+
assert dates.all?{|d| d <= yesterday}
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
33
63
|
class ExportTest < self
|
34
64
|
def setup
|
35
65
|
super
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-mixpanel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yoshihara
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-09-
|
12
|
+
date: 2016-09-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|