embulk-input-mixpanel 0.4.4 → 0.4.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f5cb5156064ae0192f6d1d2321a173cd12ef5fd3
4
- data.tar.gz: 2a2f3a63e55035bb8dd96a8edf27eb45e546866d
3
+ metadata.gz: 3c39e9af49ffa9cfcc15fe3156896652720fb72e
4
+ data.tar.gz: 350093593fbd3f62c80a10071cdcb473f36203e3
5
5
  SHA512:
6
- metadata.gz: 48d7e7af0b5ad28fc030e7c1baaa4c34fd236b60fb34df7e8cc1b764825c143aadcab7725d4d78afe3b1ba0ffb05067418217872d79cc0a1d00af7440e7abd90
7
- data.tar.gz: b001c794683c39d69ae6b29782988ee3c64ea17a9449d29fb7d86342b144854116116b1928946f33691eaaf0f3ed6d711f985fbe7e4a414c7579a3fd6baf38ec
6
+ metadata.gz: a7fe90abd0541686bb1dc4e5426a65524d93d02d4781d3f96b30d2cc8db97231302df64a11db378279a665bafeb086712d0bc3f465588f2e0bb0d9003e8cd34f
7
+ data.tar.gz: 89953f9317aae738fc331f878dde1096cfa7e53f8ef869081957306cf9fec5097f10d7affe9a19e467a5f22bf91b8af72b087aa136f550f55ed1e93ff4f7bcf9
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 0.4.5 - 2016-09-05
2
+ * [fixed] Don't try to guess future date [#43](https://github.com/treasure-data/embulk-input-mixpanel/pull/43)
3
+
1
4
  ## 0.4.4 - 2016-09-02
2
5
  * [enhancement] Reduce memory usage by streaming processing [#42](https://github.com/treasure-data/embulk-input-mixpanel/pull/42)
3
6
 
@@ -1,7 +1,7 @@
1
1
 
2
2
  Gem::Specification.new do |spec|
3
3
  spec.name = "embulk-input-mixpanel"
4
- spec.version = "0.4.4"
4
+ spec.version = "0.4.5"
5
5
  spec.authors = ["yoshihara", "uu59"]
6
6
  spec.summary = "Mixpanel input plugin for Embulk"
7
7
  spec.description = "Loads records from Mixpanel."
@@ -56,13 +56,11 @@ module Embulk
56
56
  end
57
57
 
58
58
  def export_for_small_dataset(params = {})
59
- try_to_dates = 5.times.map do |n|
60
- # from_date + 1, from_date + 10, from_date + 100, ... so on
61
- days = 1 * (10 ** n)
62
- Date.parse(params["from_date"].to_s) + days
63
- end
64
-
65
- try_to_dates.each do |to_date|
59
+ yesterday = Date.today - 1
60
+ latest_tried_to_date = nil
61
+ try_to_dates(params["from_date"]).each do |to_date|
62
+ next if yesterday < to_date
63
+ latest_tried_to_date = to_date
66
64
  params["to_date"] = to_date.strftime("%Y-%m-%d")
67
65
  records = retryer.with_retry do
68
66
  request_small_dataset(params, SMALLSET_BYTE_RANGE)
@@ -71,7 +69,18 @@ module Embulk
71
69
  return records
72
70
  end
73
71
 
74
- raise ConfigError.new "#{params["from_date"]}..#{try_to_dates.last} has no record. too old date?"
72
+ raise ConfigError.new "#{params["from_date"]}..#{latest_tried_to_date} has no record."
73
+ end
74
+
75
+ def try_to_dates(from_date)
76
+ try_to_dates = 5.times.map do |n|
77
+ # from_date + 1, from_date + 10, from_date + 100, ... so on
78
+ days = 1 * (10 ** n)
79
+ Date.parse(from_date.to_s) + days
80
+ end
81
+ yesterday = Date.today - 1
82
+ try_to_dates << yesterday
83
+ try_to_dates.find_all {|date| date <= yesterday}.uniq
75
84
  end
76
85
 
77
86
  private
@@ -30,6 +30,36 @@ module Embulk
30
30
  assert_equal(expected, @client.__send__(:signature, params))
31
31
  end
32
32
 
33
+ class TryToDatesTest < self
34
+ def setup
35
+ @client = Client.new(API_KEY, API_SECRET)
36
+ end
37
+
38
+ data do
39
+ [
40
+ ["2000-01-01", "2000-01-01"],
41
+ ["2010-01-01", "2010-01-01"],
42
+ ["3 days ago", (Date.today - 3).to_s],
43
+ ]
44
+ end
45
+ def test_candidates(from_str)
46
+ from = Date.parse(from_str)
47
+ yesterday = Date.today - 1
48
+ dates = @client.try_to_dates(from.to_s)
49
+ expect = [
50
+ from + 1,
51
+ from + 10,
52
+ from + 100,
53
+ from + 1000,
54
+ from + 10000,
55
+ yesterday,
56
+ ].find_all{|d| d <= yesterday}
57
+
58
+ assert_equal expect, dates
59
+ assert dates.all?{|d| d <= yesterday}
60
+ end
61
+ end
62
+
33
63
  class ExportTest < self
34
64
  def setup
35
65
  super
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-mixpanel
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.4
4
+ version: 0.4.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - yoshihara
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2016-09-02 00:00:00.000000000 Z
12
+ date: 2016-09-05 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement