embulk-input-mixpanel 0.5.11 → 0.5.12.alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6595e8e2cafe56340ff599d371303c45b868d70c
4
- data.tar.gz: 96c82be9851fc2008b04a352294a7ad4e60ebcb9
3
+ metadata.gz: 62f37b674cb82550f2961ccf1e13ace51ea57fc2
4
+ data.tar.gz: b62127bec5d8c43b7998fdfce6f2dc560c29b343
5
5
  SHA512:
6
- metadata.gz: ae2d8ad4e060e65a1ec37ba02e95603df8157448f3deeb04c8851ec1c2f2f4229e228b0a1cfbf74f4633a939adc1319d79404742d39543f75e879a8372210f76
7
- data.tar.gz: e27412b27786522a2e3e2419f52be463cea2c7993814b0a375cce524901a6139ca1f7f27923746e04c54f6bfa8516edc75c23548135360f1e2375fe2ce7af248
6
+ metadata.gz: 240e77ee4ea62b38bb9f2cec3fb0a12019d16852efd4976106ea5175fd2d4121a18ebb4ed794ca27ee91ef999afe2ec0d19a1eaf7da2ccdb628c09aa28b1cdd1
7
+ data.tar.gz: eb44642ae282601ea8aea26067ad226b4235fa3b06bdeb35378d8ca19ee53406349bd6110a419cb67240120dc2f78b5d3ca8ebf50c853779a8246839b3101e85
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = "embulk-input-mixpanel"
3
- spec.version = "0.5.11"
3
+ spec.version = "0.5.12.alpha.1"
4
4
  spec.authors = ["yoshihara", "uu59"]
5
5
  spec.summary = "Mixpanel input plugin for Embulk"
6
6
  spec.description = "Loads records from Mixpanel."
@@ -15,6 +15,7 @@ Gem::Specification.new do |spec|
15
15
  spec.add_dependency 'httpclient', '>= 2.8.3' # To use tcp_keepalive
16
16
  spec.add_dependency 'tzinfo'
17
17
  spec.add_dependency 'perfect_retry', ["~> 0.5"]
18
+ spec.add_dependency 'activesupport'
18
19
  spec.add_development_dependency 'bundler', ['~> 1.0']
19
20
  spec.add_development_dependency 'rake', ['>= 10.0']
20
21
  spec.add_development_dependency 'embulk', ['>= 0.8.6', '< 1.0']
@@ -4,6 +4,7 @@ require "embulk/input/mixpanel_api/client"
4
4
  require "embulk/input/mixpanel_api/exceptions"
5
5
  require "range_generator"
6
6
  require "timezone_validator"
7
+ require "active_support/core_ext/time"
7
8
 
8
9
  module Embulk
9
10
  module Input
@@ -33,7 +34,7 @@ module Embulk
33
34
  timezone = config.param(:timezone, :string)
34
35
  TimezoneValidator.new(timezone).validate
35
36
 
36
- from_date = config.param(:from_date, :string, default: (Date.today - 2).to_s)
37
+ from_date = config.param(:from_date, :string, default: (today(timezone) - 2).to_s)
37
38
  fetch_days = config.param(:fetch_days, :integer, default: nil)
38
39
 
39
40
 
@@ -51,7 +52,7 @@ module Embulk
51
52
  fetch_days = fetch_days.nil? ? nil : fetch_days + back_fill_days
52
53
  end
53
54
 
54
- range = RangeGenerator.new(from_date, fetch_days).generate_range
55
+ range = RangeGenerator.new(from_date, fetch_days, timezone).generate_range
55
56
  Embulk.logger.info "Try to fetch data from #{range.first} to #{range.last}"
56
57
  job_start_time = Time.now.to_i*1000
57
58
  upper_limit_delay = config.param(:incremental_column_upper_limit_delay_in_seconds, :integer, default: 0)
@@ -228,7 +229,7 @@ module Embulk
228
229
  page_builder.finish
229
230
  task_report = {
230
231
  latest_fetched_time: current_latest_fetched_time,
231
- to_date: @dates.last || Date.today - 1,
232
+ to_date: @dates.last || today(@timezone) - 1,
232
233
  }
233
234
  task_report
234
235
  end
@@ -338,16 +339,17 @@ module Embulk
338
339
  }
339
340
  end
340
341
 
341
- def self.default_guess_start_date
342
- Date.today - DEFAULT_FETCH_DAYS - 1
342
+ def self.default_guess_start_date(timezone)
343
+ today(timezone) - DEFAULT_FETCH_DAYS - 1
343
344
  end
344
345
 
345
346
  def self.guess_range(config)
346
- from_date = config.param(:from_date, :string, default: default_guess_start_date.to_s)
347
+ time_zone = config.param(:timezone, :string, default: "")
348
+ from_date = config.param(:from_date, :string, default: default_guess_start_date(time_zone).to_s)
347
349
  fetch_days = config.param(:fetch_days, :integer, default: DEFAULT_FETCH_DAYS)
348
- range = RangeGenerator.new(from_date, fetch_days).generate_range
350
+ range = RangeGenerator.new(from_date, fetch_days, time_zone).generate_range
349
351
  if range.empty?
350
- return default_guess_start_date..(Date.today - 1)
352
+ return default_guess_start_date(time_zone)..(today(time_zone) - 1)
351
353
  end
352
354
  range
353
355
  end
@@ -368,7 +370,16 @@ module Embulk
368
370
  # Shift incremental column to top
369
371
  columns.unshift(name: "time", type: :long)
370
372
  end
371
- end
372
373
 
374
+ def self.today(timezone)
375
+ if timezone.nil?
376
+ Date.today
377
+ else
378
+ zone = ActiveSupport::TimeZone[timezone]
379
+ zone.nil? ? Date.today : zone.today
380
+ end
381
+ end
382
+
383
+ end
373
384
  end
374
385
  end
@@ -1,9 +1,12 @@
1
+ require "active_support/core_ext/time"
2
+
1
3
  class RangeGenerator
2
- attr_reader :from_date_str, :fetch_days
4
+ attr_reader :from_date_str, :fetch_days, :profile_time_zone
3
5
 
4
- def initialize(from_date_str, fetch_days)
6
+ def initialize(from_date_str, fetch_days, profile_time_zone)
5
7
  @from_date_str = from_date_str
6
8
  @fetch_days = fetch_days
9
+ @profile_time_zone = profile_time_zone
7
10
  end
8
11
 
9
12
  def generate_range
@@ -70,6 +73,8 @@ class RangeGenerator
70
73
  end
71
74
 
72
75
  def today
73
- @today ||= Date.today
76
+ zone = ActiveSupport::TimeZone[@profile_time_zone]
77
+ Embulk.logger.warn "Coudn't find timezone support for #{@profile_time_zone}" if zone.nil?
78
+ @today ||= (zone.nil? ? Date.today : zone.today)
74
79
  end
75
80
  end
@@ -1,6 +1,7 @@
1
1
  require "prepare_embulk"
2
2
  require "override_assert_raise"
3
3
  require "embulk/input/mixpanel"
4
+ require "active_support/core_ext/time"
4
5
  require "json"
5
6
 
6
7
  module Embulk
@@ -88,17 +89,18 @@ module Embulk
88
89
  type: "mixpanel",
89
90
  api_key: API_KEY,
90
91
  api_secret: API_SECRET,
91
- from_date: (Date.today + 1).to_s,
92
+ timezone: TIMEZONE,
93
+ from_date: (today + 1).to_s
92
94
  }
93
95
 
94
96
  stub_export_all
95
- mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date.to_s}/)
97
+ mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date(TIMEZONE).to_s}/)
96
98
 
97
99
  Mixpanel.guess(embulk_config(config))
98
100
  end
99
101
 
100
102
  def test_from_date_yesterday
101
- from_date = (Date.today - 1).to_s
103
+ from_date = (today - 1).to_s
102
104
  config = {
103
105
  type: "mixpanel",
104
106
  api_key: API_KEY,
@@ -117,10 +119,11 @@ module Embulk
117
119
  type: "mixpanel",
118
120
  api_key: API_KEY,
119
121
  api_secret: API_SECRET,
122
+ timezone: TIMEZONE
120
123
  }
121
124
 
122
125
  stub_export_all
123
- mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date.to_s}/)
126
+ mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date(TIMEZONE).to_s}/)
124
127
 
125
128
  Mixpanel.guess(embulk_config(config))
126
129
  end
@@ -192,7 +195,7 @@ module Embulk
192
195
  end
193
196
 
194
197
  def test_future
195
- from_date = (Date.today + 10).to_s
198
+ from_date = (today + 10).to_s
196
199
  mock(Mixpanel).resume(anything, anything, 1)
197
200
 
198
201
  Mixpanel.transaction(transaction_config(from_date))
@@ -200,7 +203,7 @@ module Embulk
200
203
 
201
204
  def test_negative_days
202
205
  assert_raise(Embulk::ConfigError) do
203
- Mixpanel.transaction(transaction_config((Date.today - 1).to_s).merge(fetch_days: -1))
206
+ Mixpanel.transaction(transaction_config((today - 1).to_s).merge(fetch_days: -1))
204
207
  end
205
208
  end
206
209
 
@@ -209,7 +212,7 @@ module Embulk
209
212
  assert_nil(task[:incremental_column])
210
213
  assert_true(task[:incremental])
211
214
  }
212
- Mixpanel.transaction(transaction_config(Date.today))
215
+ Mixpanel.transaction(transaction_config(today))
213
216
  end
214
217
 
215
218
  private
@@ -259,11 +262,11 @@ module Embulk
259
262
  private
260
263
 
261
264
  def dates
262
- (Date.today - 10)..(Date.today + 10)
265
+ (today - 10)..(today + 10)
263
266
  end
264
267
 
265
268
  def target_dates
266
- dates.find_all{|d| d <= Date.today}.map {|date| date.to_s}
269
+ dates.find_all{|d| d <= today}.map {|date| date.to_s}
267
270
  end
268
271
 
269
272
  def transaction_config
@@ -325,9 +328,9 @@ module Embulk
325
328
 
326
329
  def test_next_to_date
327
330
  next_config_diff = Mixpanel.resume(transaction_task(1).merge(incremental: true), columns, 1) do
328
- [{to_date: Date.today.to_s, latest_fetched_time: 1502707247000}]
331
+ [{to_date: today.to_s, latest_fetched_time: 1502707247000}]
329
332
  end
330
- assert_equal(Date.today.to_s, next_config_diff[:from_date])
333
+ assert_equal(today.to_s, next_config_diff[:from_date])
331
334
  end
332
335
 
333
336
  def test_valid_days_with_backfill
@@ -419,7 +422,6 @@ module Embulk
419
422
  end
420
423
 
421
424
  def test_resume
422
- today = Date.today
423
425
  control = proc { [{to_date: today.to_s, latest_fetched_time: 999}] }
424
426
  actual = Mixpanel.resume(transaction_task, columns, 1, &control)
425
427
  assert_equal({from_date: today.to_s, latest_fetched_time: 999}, actual)
@@ -905,6 +907,9 @@ module Embulk
905
907
  def embulk_config
906
908
  DataSource[*config.to_a.flatten(1)]
907
909
  end
910
+ def today
911
+ ActiveSupport::TimeZone[TIMEZONE].today
912
+ end
908
913
  end
909
914
  end
910
915
  end
@@ -1,9 +1,11 @@
1
1
  require "range_generator"
2
2
  require "override_assert_raise"
3
+ require "active_support/core_ext/time"
3
4
 
4
5
  class RangeGeneratorTest < Test::Unit::TestCase
5
6
  include OverrideAssertRaise
6
-
7
+ DEFAULT_TIMEZONE = "America/Chicago"
8
+ DEFAULT_LOCAL = ActiveSupport::TimeZone["UTC"]
7
9
  class GenerateRangeTest < self
8
10
  data do
9
11
  {
@@ -25,7 +27,7 @@ class RangeGeneratorTest < Test::Unit::TestCase
25
27
 
26
28
  expected = (expected_from..expected_to).to_a.map{|date| date.to_s}
27
29
 
28
- actual = RangeGenerator.new(from, days).generate_range
30
+ actual = RangeGenerator.new(from, days, DEFAULT_TIMEZONE).generate_range
29
31
 
30
32
  assert_equal(expected, actual)
31
33
  end
@@ -36,7 +38,7 @@ class RangeGeneratorTest < Test::Unit::TestCase
36
38
  expected_from = Date.parse(from)
37
39
  expected_to = Date.parse("2017-08-04")
38
40
  expected = (expected_from..expected_to).to_a.map {|date| date.to_s}
39
- actual = RangeGenerator.new(from, days).generate_range
41
+ actual = RangeGenerator.new(from, days, DEFAULT_TIMEZONE).generate_range
40
42
  assert_equal(expected, actual)
41
43
  end
42
44
 
@@ -46,19 +48,20 @@ class RangeGeneratorTest < Test::Unit::TestCase
46
48
  expected_from = Date.parse(from)
47
49
  expected_to = Date.parse("2017-08-08")
48
50
  expected = (expected_from..expected_to).to_a.map {|date| date.to_s}
49
- actual = RangeGenerator.new(from, days).generate_range
51
+ actual = RangeGenerator.new(from, days, DEFAULT_TIMEZONE).generate_range
50
52
  assert_equal(expected, actual)
51
53
  end
52
54
 
53
55
  class OverDaysTest < self
54
56
  def setup
55
- @from = Date.today - 5
57
+ @from = today - 5
56
58
  @days = 10
57
59
  @warn_message_regexp = /ignored them/
58
60
  end
59
61
 
60
62
  def test_range_only_present
61
- expected_to = Date.today
63
+
64
+ expected_to = today
62
65
  expected = (@from..expected_to).to_a.map{|date| date.to_s}
63
66
 
64
67
  stub(Embulk.logger).warn(@warn_message_regexp)
@@ -81,7 +84,7 @@ class RangeGeneratorTest < Test::Unit::TestCase
81
84
 
82
85
  class FromDateEarlyTest < self
83
86
  def setup
84
- @from = Date.today + 5
87
+ @from = today + 5
85
88
  @days = 10
86
89
  @warn_message_regexp = /allow 2 days/
87
90
  end
@@ -108,7 +111,10 @@ class RangeGeneratorTest < Test::Unit::TestCase
108
111
  private
109
112
 
110
113
  def generate_range(from_date_str, fetch_days)
111
- RangeGenerator.new(from_date_str, fetch_days).generate_range
114
+ RangeGenerator.new(from_date_str, fetch_days, DEFAULT_TIMEZONE).generate_range
115
+ end
116
+ def today
117
+ DEFAULT_LOCAL.today
112
118
  end
113
119
  end
114
120
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-mixpanel
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.11
4
+ version: 0.5.12.alpha.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - yoshihara
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-12-11 00:00:00.000000000 Z
12
+ date: 2018-02-05 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -53,6 +53,20 @@ dependencies:
53
53
  - - "~>"
54
54
  - !ruby/object:Gem::Version
55
55
  version: '0.5'
56
+ - !ruby/object:Gem::Dependency
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ name: activesupport
63
+ prerelease: false
64
+ type: :runtime
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
56
70
  - !ruby/object:Gem::Dependency
57
71
  requirement: !ruby/object:Gem::Requirement
58
72
  requirements:
@@ -220,12 +234,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
220
234
  version: '0'
221
235
  required_rubygems_version: !ruby/object:Gem::Requirement
222
236
  requirements:
223
- - - ">="
237
+ - - ">"
224
238
  - !ruby/object:Gem::Version
225
- version: '0'
239
+ version: 1.3.1
226
240
  requirements: []
227
241
  rubyforge_project:
228
- rubygems_version: 2.4.8
242
+ rubygems_version: 2.6.6
229
243
  signing_key:
230
244
  specification_version: 4
231
245
  summary: Mixpanel input plugin for Embulk