embulk-input-mixpanel 0.5.11 → 0.5.12.alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/embulk-input-mixpanel.gemspec +2 -1
- data/lib/embulk/input/mixpanel.rb +20 -9
- data/lib/range_generator.rb +8 -3
- data/test/embulk/input/test_mixpanel.rb +17 -12
- data/test/test_range_generator.rb +14 -8
- metadata +19 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 62f37b674cb82550f2961ccf1e13ace51ea57fc2
|
4
|
+
data.tar.gz: b62127bec5d8c43b7998fdfce6f2dc560c29b343
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 240e77ee4ea62b38bb9f2cec3fb0a12019d16852efd4976106ea5175fd2d4121a18ebb4ed794ca27ee91ef999afe2ec0d19a1eaf7da2ccdb628c09aa28b1cdd1
|
7
|
+
data.tar.gz: eb44642ae282601ea8aea26067ad226b4235fa3b06bdeb35378d8ca19ee53406349bd6110a419cb67240120dc2f78b5d3ca8ebf50c853779a8246839b3101e85
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "embulk-input-mixpanel"
|
3
|
-
spec.version = "0.5.
|
3
|
+
spec.version = "0.5.12.alpha.1"
|
4
4
|
spec.authors = ["yoshihara", "uu59"]
|
5
5
|
spec.summary = "Mixpanel input plugin for Embulk"
|
6
6
|
spec.description = "Loads records from Mixpanel."
|
@@ -15,6 +15,7 @@ Gem::Specification.new do |spec|
|
|
15
15
|
spec.add_dependency 'httpclient', '>= 2.8.3' # To use tcp_keepalive
|
16
16
|
spec.add_dependency 'tzinfo'
|
17
17
|
spec.add_dependency 'perfect_retry', ["~> 0.5"]
|
18
|
+
spec.add_dependency 'activesupport'
|
18
19
|
spec.add_development_dependency 'bundler', ['~> 1.0']
|
19
20
|
spec.add_development_dependency 'rake', ['>= 10.0']
|
20
21
|
spec.add_development_dependency 'embulk', ['>= 0.8.6', '< 1.0']
|
@@ -4,6 +4,7 @@ require "embulk/input/mixpanel_api/client"
|
|
4
4
|
require "embulk/input/mixpanel_api/exceptions"
|
5
5
|
require "range_generator"
|
6
6
|
require "timezone_validator"
|
7
|
+
require "active_support/core_ext/time"
|
7
8
|
|
8
9
|
module Embulk
|
9
10
|
module Input
|
@@ -33,7 +34,7 @@ module Embulk
|
|
33
34
|
timezone = config.param(:timezone, :string)
|
34
35
|
TimezoneValidator.new(timezone).validate
|
35
36
|
|
36
|
-
from_date = config.param(:from_date, :string, default: (
|
37
|
+
from_date = config.param(:from_date, :string, default: (today(timezone) - 2).to_s)
|
37
38
|
fetch_days = config.param(:fetch_days, :integer, default: nil)
|
38
39
|
|
39
40
|
|
@@ -51,7 +52,7 @@ module Embulk
|
|
51
52
|
fetch_days = fetch_days.nil? ? nil : fetch_days + back_fill_days
|
52
53
|
end
|
53
54
|
|
54
|
-
range = RangeGenerator.new(from_date, fetch_days).generate_range
|
55
|
+
range = RangeGenerator.new(from_date, fetch_days, timezone).generate_range
|
55
56
|
Embulk.logger.info "Try to fetch data from #{range.first} to #{range.last}"
|
56
57
|
job_start_time = Time.now.to_i*1000
|
57
58
|
upper_limit_delay = config.param(:incremental_column_upper_limit_delay_in_seconds, :integer, default: 0)
|
@@ -228,7 +229,7 @@ module Embulk
|
|
228
229
|
page_builder.finish
|
229
230
|
task_report = {
|
230
231
|
latest_fetched_time: current_latest_fetched_time,
|
231
|
-
to_date: @dates.last ||
|
232
|
+
to_date: @dates.last || today(@timezone) - 1,
|
232
233
|
}
|
233
234
|
task_report
|
234
235
|
end
|
@@ -338,16 +339,17 @@ module Embulk
|
|
338
339
|
}
|
339
340
|
end
|
340
341
|
|
341
|
-
def self.default_guess_start_date
|
342
|
-
|
342
|
+
def self.default_guess_start_date(timezone)
|
343
|
+
today(timezone) - DEFAULT_FETCH_DAYS - 1
|
343
344
|
end
|
344
345
|
|
345
346
|
def self.guess_range(config)
|
346
|
-
|
347
|
+
time_zone = config.param(:timezone, :string, default: "")
|
348
|
+
from_date = config.param(:from_date, :string, default: default_guess_start_date(time_zone).to_s)
|
347
349
|
fetch_days = config.param(:fetch_days, :integer, default: DEFAULT_FETCH_DAYS)
|
348
|
-
range = RangeGenerator.new(from_date, fetch_days).generate_range
|
350
|
+
range = RangeGenerator.new(from_date, fetch_days, time_zone).generate_range
|
349
351
|
if range.empty?
|
350
|
-
return default_guess_start_date..(
|
352
|
+
return default_guess_start_date(time_zone)..(today(time_zone) - 1)
|
351
353
|
end
|
352
354
|
range
|
353
355
|
end
|
@@ -368,7 +370,16 @@ module Embulk
|
|
368
370
|
# Shift incremental column to top
|
369
371
|
columns.unshift(name: "time", type: :long)
|
370
372
|
end
|
371
|
-
end
|
372
373
|
|
374
|
+
def self.today(timezone)
|
375
|
+
if timezone.nil?
|
376
|
+
Date.today
|
377
|
+
else
|
378
|
+
zone = ActiveSupport::TimeZone[timezone]
|
379
|
+
zone.nil? ? Date.today : zone.today
|
380
|
+
end
|
381
|
+
end
|
382
|
+
|
383
|
+
end
|
373
384
|
end
|
374
385
|
end
|
data/lib/range_generator.rb
CHANGED
@@ -1,9 +1,12 @@
|
|
1
|
+
require "active_support/core_ext/time"
|
2
|
+
|
1
3
|
class RangeGenerator
|
2
|
-
attr_reader :from_date_str, :fetch_days
|
4
|
+
attr_reader :from_date_str, :fetch_days, :profile_time_zone
|
3
5
|
|
4
|
-
def initialize(from_date_str, fetch_days)
|
6
|
+
def initialize(from_date_str, fetch_days, profile_time_zone)
|
5
7
|
@from_date_str = from_date_str
|
6
8
|
@fetch_days = fetch_days
|
9
|
+
@profile_time_zone = profile_time_zone
|
7
10
|
end
|
8
11
|
|
9
12
|
def generate_range
|
@@ -70,6 +73,8 @@ class RangeGenerator
|
|
70
73
|
end
|
71
74
|
|
72
75
|
def today
|
73
|
-
|
76
|
+
zone = ActiveSupport::TimeZone[@profile_time_zone]
|
77
|
+
Embulk.logger.warn "Coudn't find timezone support for #{@profile_time_zone}" if zone.nil?
|
78
|
+
@today ||= (zone.nil? ? Date.today : zone.today)
|
74
79
|
end
|
75
80
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require "prepare_embulk"
|
2
2
|
require "override_assert_raise"
|
3
3
|
require "embulk/input/mixpanel"
|
4
|
+
require "active_support/core_ext/time"
|
4
5
|
require "json"
|
5
6
|
|
6
7
|
module Embulk
|
@@ -88,17 +89,18 @@ module Embulk
|
|
88
89
|
type: "mixpanel",
|
89
90
|
api_key: API_KEY,
|
90
91
|
api_secret: API_SECRET,
|
91
|
-
|
92
|
+
timezone: TIMEZONE,
|
93
|
+
from_date: (today + 1).to_s
|
92
94
|
}
|
93
95
|
|
94
96
|
stub_export_all
|
95
|
-
mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date.to_s}/)
|
97
|
+
mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date(TIMEZONE).to_s}/)
|
96
98
|
|
97
99
|
Mixpanel.guess(embulk_config(config))
|
98
100
|
end
|
99
101
|
|
100
102
|
def test_from_date_yesterday
|
101
|
-
from_date = (
|
103
|
+
from_date = (today - 1).to_s
|
102
104
|
config = {
|
103
105
|
type: "mixpanel",
|
104
106
|
api_key: API_KEY,
|
@@ -117,10 +119,11 @@ module Embulk
|
|
117
119
|
type: "mixpanel",
|
118
120
|
api_key: API_KEY,
|
119
121
|
api_secret: API_SECRET,
|
122
|
+
timezone: TIMEZONE
|
120
123
|
}
|
121
124
|
|
122
125
|
stub_export_all
|
123
|
-
mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date.to_s}/)
|
126
|
+
mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date(TIMEZONE).to_s}/)
|
124
127
|
|
125
128
|
Mixpanel.guess(embulk_config(config))
|
126
129
|
end
|
@@ -192,7 +195,7 @@ module Embulk
|
|
192
195
|
end
|
193
196
|
|
194
197
|
def test_future
|
195
|
-
from_date = (
|
198
|
+
from_date = (today + 10).to_s
|
196
199
|
mock(Mixpanel).resume(anything, anything, 1)
|
197
200
|
|
198
201
|
Mixpanel.transaction(transaction_config(from_date))
|
@@ -200,7 +203,7 @@ module Embulk
|
|
200
203
|
|
201
204
|
def test_negative_days
|
202
205
|
assert_raise(Embulk::ConfigError) do
|
203
|
-
Mixpanel.transaction(transaction_config((
|
206
|
+
Mixpanel.transaction(transaction_config((today - 1).to_s).merge(fetch_days: -1))
|
204
207
|
end
|
205
208
|
end
|
206
209
|
|
@@ -209,7 +212,7 @@ module Embulk
|
|
209
212
|
assert_nil(task[:incremental_column])
|
210
213
|
assert_true(task[:incremental])
|
211
214
|
}
|
212
|
-
Mixpanel.transaction(transaction_config(
|
215
|
+
Mixpanel.transaction(transaction_config(today))
|
213
216
|
end
|
214
217
|
|
215
218
|
private
|
@@ -259,11 +262,11 @@ module Embulk
|
|
259
262
|
private
|
260
263
|
|
261
264
|
def dates
|
262
|
-
(
|
265
|
+
(today - 10)..(today + 10)
|
263
266
|
end
|
264
267
|
|
265
268
|
def target_dates
|
266
|
-
dates.find_all{|d| d <=
|
269
|
+
dates.find_all{|d| d <= today}.map {|date| date.to_s}
|
267
270
|
end
|
268
271
|
|
269
272
|
def transaction_config
|
@@ -325,9 +328,9 @@ module Embulk
|
|
325
328
|
|
326
329
|
def test_next_to_date
|
327
330
|
next_config_diff = Mixpanel.resume(transaction_task(1).merge(incremental: true), columns, 1) do
|
328
|
-
[{to_date:
|
331
|
+
[{to_date: today.to_s, latest_fetched_time: 1502707247000}]
|
329
332
|
end
|
330
|
-
assert_equal(
|
333
|
+
assert_equal(today.to_s, next_config_diff[:from_date])
|
331
334
|
end
|
332
335
|
|
333
336
|
def test_valid_days_with_backfill
|
@@ -419,7 +422,6 @@ module Embulk
|
|
419
422
|
end
|
420
423
|
|
421
424
|
def test_resume
|
422
|
-
today = Date.today
|
423
425
|
control = proc { [{to_date: today.to_s, latest_fetched_time: 999}] }
|
424
426
|
actual = Mixpanel.resume(transaction_task, columns, 1, &control)
|
425
427
|
assert_equal({from_date: today.to_s, latest_fetched_time: 999}, actual)
|
@@ -905,6 +907,9 @@ module Embulk
|
|
905
907
|
def embulk_config
|
906
908
|
DataSource[*config.to_a.flatten(1)]
|
907
909
|
end
|
910
|
+
def today
|
911
|
+
ActiveSupport::TimeZone[TIMEZONE].today
|
912
|
+
end
|
908
913
|
end
|
909
914
|
end
|
910
915
|
end
|
@@ -1,9 +1,11 @@
|
|
1
1
|
require "range_generator"
|
2
2
|
require "override_assert_raise"
|
3
|
+
require "active_support/core_ext/time"
|
3
4
|
|
4
5
|
class RangeGeneratorTest < Test::Unit::TestCase
|
5
6
|
include OverrideAssertRaise
|
6
|
-
|
7
|
+
DEFAULT_TIMEZONE = "America/Chicago"
|
8
|
+
DEFAULT_LOCAL = ActiveSupport::TimeZone["UTC"]
|
7
9
|
class GenerateRangeTest < self
|
8
10
|
data do
|
9
11
|
{
|
@@ -25,7 +27,7 @@ class RangeGeneratorTest < Test::Unit::TestCase
|
|
25
27
|
|
26
28
|
expected = (expected_from..expected_to).to_a.map{|date| date.to_s}
|
27
29
|
|
28
|
-
actual = RangeGenerator.new(from, days).generate_range
|
30
|
+
actual = RangeGenerator.new(from, days, DEFAULT_TIMEZONE).generate_range
|
29
31
|
|
30
32
|
assert_equal(expected, actual)
|
31
33
|
end
|
@@ -36,7 +38,7 @@ class RangeGeneratorTest < Test::Unit::TestCase
|
|
36
38
|
expected_from = Date.parse(from)
|
37
39
|
expected_to = Date.parse("2017-08-04")
|
38
40
|
expected = (expected_from..expected_to).to_a.map {|date| date.to_s}
|
39
|
-
actual = RangeGenerator.new(from, days).generate_range
|
41
|
+
actual = RangeGenerator.new(from, days, DEFAULT_TIMEZONE).generate_range
|
40
42
|
assert_equal(expected, actual)
|
41
43
|
end
|
42
44
|
|
@@ -46,19 +48,20 @@ class RangeGeneratorTest < Test::Unit::TestCase
|
|
46
48
|
expected_from = Date.parse(from)
|
47
49
|
expected_to = Date.parse("2017-08-08")
|
48
50
|
expected = (expected_from..expected_to).to_a.map {|date| date.to_s}
|
49
|
-
actual = RangeGenerator.new(from, days).generate_range
|
51
|
+
actual = RangeGenerator.new(from, days, DEFAULT_TIMEZONE).generate_range
|
50
52
|
assert_equal(expected, actual)
|
51
53
|
end
|
52
54
|
|
53
55
|
class OverDaysTest < self
|
54
56
|
def setup
|
55
|
-
@from =
|
57
|
+
@from = today - 5
|
56
58
|
@days = 10
|
57
59
|
@warn_message_regexp = /ignored them/
|
58
60
|
end
|
59
61
|
|
60
62
|
def test_range_only_present
|
61
|
-
|
63
|
+
|
64
|
+
expected_to = today
|
62
65
|
expected = (@from..expected_to).to_a.map{|date| date.to_s}
|
63
66
|
|
64
67
|
stub(Embulk.logger).warn(@warn_message_regexp)
|
@@ -81,7 +84,7 @@ class RangeGeneratorTest < Test::Unit::TestCase
|
|
81
84
|
|
82
85
|
class FromDateEarlyTest < self
|
83
86
|
def setup
|
84
|
-
@from =
|
87
|
+
@from = today + 5
|
85
88
|
@days = 10
|
86
89
|
@warn_message_regexp = /allow 2 days/
|
87
90
|
end
|
@@ -108,7 +111,10 @@ class RangeGeneratorTest < Test::Unit::TestCase
|
|
108
111
|
private
|
109
112
|
|
110
113
|
def generate_range(from_date_str, fetch_days)
|
111
|
-
RangeGenerator.new(from_date_str, fetch_days).generate_range
|
114
|
+
RangeGenerator.new(from_date_str, fetch_days, DEFAULT_TIMEZONE).generate_range
|
115
|
+
end
|
116
|
+
def today
|
117
|
+
DEFAULT_LOCAL.today
|
112
118
|
end
|
113
119
|
end
|
114
120
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-mixpanel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.12.alpha.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yoshihara
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2018-02-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -53,6 +53,20 @@ dependencies:
|
|
53
53
|
- - "~>"
|
54
54
|
- !ruby/object:Gem::Version
|
55
55
|
version: '0.5'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
name: activesupport
|
63
|
+
prerelease: false
|
64
|
+
type: :runtime
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
56
70
|
- !ruby/object:Gem::Dependency
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
58
72
|
requirements:
|
@@ -220,12 +234,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
220
234
|
version: '0'
|
221
235
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
222
236
|
requirements:
|
223
|
-
- - "
|
237
|
+
- - ">"
|
224
238
|
- !ruby/object:Gem::Version
|
225
|
-
version:
|
239
|
+
version: 1.3.1
|
226
240
|
requirements: []
|
227
241
|
rubyforge_project:
|
228
|
-
rubygems_version: 2.
|
242
|
+
rubygems_version: 2.6.6
|
229
243
|
signing_key:
|
230
244
|
specification_version: 4
|
231
245
|
summary: Mixpanel input plugin for Embulk
|