embulk-input-mixpanel 0.4.4 → 0.4.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/embulk-input-mixpanel.gemspec +1 -1
- data/lib/embulk/input/mixpanel_api/client.rb +17 -8
- data/test/embulk/input/mixpanel_api/test_client.rb +30 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3c39e9af49ffa9cfcc15fe3156896652720fb72e
|
4
|
+
data.tar.gz: 350093593fbd3f62c80a10071cdcb473f36203e3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a7fe90abd0541686bb1dc4e5426a65524d93d02d4781d3f96b30d2cc8db97231302df64a11db378279a665bafeb086712d0bc3f465588f2e0bb0d9003e8cd34f
|
7
|
+
data.tar.gz: 89953f9317aae738fc331f878dde1096cfa7e53f8ef869081957306cf9fec5097f10d7affe9a19e467a5f22bf91b8af72b087aa136f550f55ed1e93ff4f7bcf9
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 0.4.5 - 2016-09-05
|
2
|
+
* [fixed] Don't try to guess future date [#43](https://github.com/treasure-data/embulk-input-mixpanel/pull/43)
|
3
|
+
|
1
4
|
## 0.4.4 - 2016-09-02
|
2
5
|
* [enhancement] Reduce memory usage by streaming processing [#42](https://github.com/treasure-data/embulk-input-mixpanel/pull/42)
|
3
6
|
|
@@ -56,13 +56,11 @@ module Embulk
|
|
56
56
|
end
|
57
57
|
|
58
58
|
def export_for_small_dataset(params = {})
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
try_to_dates.each do |to_date|
|
59
|
+
yesterday = Date.today - 1
|
60
|
+
latest_tried_to_date = nil
|
61
|
+
try_to_dates(params["from_date"]).each do |to_date|
|
62
|
+
next if yesterday < to_date
|
63
|
+
latest_tried_to_date = to_date
|
66
64
|
params["to_date"] = to_date.strftime("%Y-%m-%d")
|
67
65
|
records = retryer.with_retry do
|
68
66
|
request_small_dataset(params, SMALLSET_BYTE_RANGE)
|
@@ -71,7 +69,18 @@ module Embulk
|
|
71
69
|
return records
|
72
70
|
end
|
73
71
|
|
74
|
-
raise ConfigError.new "#{params["from_date"]}..#{
|
72
|
+
raise ConfigError.new "#{params["from_date"]}..#{latest_tried_to_date} has no record."
|
73
|
+
end
|
74
|
+
|
75
|
+
def try_to_dates(from_date)
|
76
|
+
try_to_dates = 5.times.map do |n|
|
77
|
+
# from_date + 1, from_date + 10, from_date + 100, ... so on
|
78
|
+
days = 1 * (10 ** n)
|
79
|
+
Date.parse(from_date.to_s) + days
|
80
|
+
end
|
81
|
+
yesterday = Date.today - 1
|
82
|
+
try_to_dates << yesterday
|
83
|
+
try_to_dates.find_all {|date| date <= yesterday}.uniq
|
75
84
|
end
|
76
85
|
|
77
86
|
private
|
@@ -30,6 +30,36 @@ module Embulk
|
|
30
30
|
assert_equal(expected, @client.__send__(:signature, params))
|
31
31
|
end
|
32
32
|
|
33
|
+
class TryToDatesTest < self
|
34
|
+
def setup
|
35
|
+
@client = Client.new(API_KEY, API_SECRET)
|
36
|
+
end
|
37
|
+
|
38
|
+
data do
|
39
|
+
[
|
40
|
+
["2000-01-01", "2000-01-01"],
|
41
|
+
["2010-01-01", "2010-01-01"],
|
42
|
+
["3 days ago", (Date.today - 3).to_s],
|
43
|
+
]
|
44
|
+
end
|
45
|
+
def test_candidates(from_str)
|
46
|
+
from = Date.parse(from_str)
|
47
|
+
yesterday = Date.today - 1
|
48
|
+
dates = @client.try_to_dates(from.to_s)
|
49
|
+
expect = [
|
50
|
+
from + 1,
|
51
|
+
from + 10,
|
52
|
+
from + 100,
|
53
|
+
from + 1000,
|
54
|
+
from + 10000,
|
55
|
+
yesterday,
|
56
|
+
].find_all{|d| d <= yesterday}
|
57
|
+
|
58
|
+
assert_equal expect, dates
|
59
|
+
assert dates.all?{|d| d <= yesterday}
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
33
63
|
class ExportTest < self
|
34
64
|
def setup
|
35
65
|
super
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-mixpanel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yoshihara
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-09-
|
12
|
+
date: 2016-09-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|