embulk-input-mixpanel 0.5.6 → 0.5.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/embulk-input-mixpanel.gemspec +1 -1
- data/lib/embulk/input/mixpanel.rb +7 -7
- data/test/embulk/input/test_mixpanel.rb +27 -33
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: da76e44b6dedb0753c503b715003e6c26a530321
|
4
|
+
data.tar.gz: bd708d46d831ddb872c501623675627754d988db
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b63e44eb7a0f39f2b72f20b44e2075a943ed770073fd9740e2a097397d1d0417c11558ab7b36eb66ced530500295ec15d5846d1f487554398722c4995036fd4a
|
7
|
+
data.tar.gz: 6310dd9990715a6ab9aacf3bfcc6e377b24ed03b3841e1aaae44a709c7b2c66483fb4ea1bdffee936e09f07b164eb1aecc5def51233baa7a9f0f9ceffc641e02
|
@@ -66,7 +66,8 @@ module Embulk
|
|
66
66
|
retry_limit: config.param(:retry_limit, :integer, default: 5),
|
67
67
|
latest_fetched_time: latest_fetched_time,
|
68
68
|
incremental: incremental,
|
69
|
-
slice_range: config.param(:slice_range, :integer, default: 7)
|
69
|
+
slice_range: config.param(:slice_range, :integer, default: 7),
|
70
|
+
job_start_time: Time.now.to_i*1000
|
70
71
|
}
|
71
72
|
|
72
73
|
if task[:fetch_unknown_columns] && task[:fetch_custom_properties]
|
@@ -159,13 +160,13 @@ module Embulk
|
|
159
160
|
prev_latest_fetched_time = task[:latest_fetched_time] || 0
|
160
161
|
prev_latest_fetched_time_format = Time.at(prev_latest_fetched_time).strftime("%F %T %z")
|
161
162
|
current_latest_fetched_time = prev_latest_fetched_time
|
162
|
-
@dates.each_slice(task[:slice_range]) do |
|
163
|
+
@dates.each_slice(task[:slice_range]) do |dates|
|
163
164
|
ignored_record_count = 0
|
164
165
|
unless preview?
|
165
|
-
Embulk.logger.info "Fetching data from #{
|
166
|
+
Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
|
166
167
|
end
|
167
168
|
record_time_column=@incremental_column || DEFAULT_TIME_COLUMN
|
168
|
-
fetch(
|
169
|
+
fetch(dates, prev_latest_fetched_time).each do |record|
|
169
170
|
if @incremental
|
170
171
|
if !record["properties"].include?(record_time_column)
|
171
172
|
raise Embulk::ConfigError.new("Incremental column not exists in fetched data #{record_time_column}")
|
@@ -193,7 +194,6 @@ module Embulk
|
|
193
194
|
end
|
194
195
|
page_builder.add(values)
|
195
196
|
end
|
196
|
-
prev_latest_fetched_time = [current_latest_fetched_time, prev_latest_fetched_time].max
|
197
197
|
if ignored_record_count > 0
|
198
198
|
Embulk.logger.warn "Skipped already loaded #{ignored_record_count} records. These record times are older or equal than previous fetched record time (#{prev_latest_fetched_time} @ #{prev_latest_fetched_time_format})."
|
199
199
|
end
|
@@ -258,7 +258,7 @@ module Embulk
|
|
258
258
|
end
|
259
259
|
end
|
260
260
|
|
261
|
-
def fetch(dates,last_fetch_time, &block)
|
261
|
+
def fetch(dates, last_fetch_time, &block)
|
262
262
|
from_date = dates.first
|
263
263
|
to_date = dates.last
|
264
264
|
params = @params.merge(
|
@@ -267,7 +267,7 @@ module Embulk
|
|
267
267
|
)
|
268
268
|
if !@incremental_column.nil? && !last_fetch_time.nil? && last_fetch_time!=0 # can't do filter on time column, time column need to be filter manually.
|
269
269
|
params = params.merge(
|
270
|
-
"where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time}"
|
270
|
+
"where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time} and properties[\"#{@incremental_column}\"] < #{task[:job_start_time]}"
|
271
271
|
)
|
272
272
|
end
|
273
273
|
Embulk.logger.info "Where params is #{params["where"]}"
|
@@ -15,7 +15,7 @@ module Embulk
|
|
15
15
|
DAYS = 8
|
16
16
|
DATES = Date.parse(FROM_DATE)..(Date.parse(FROM_DATE) + DAYS - 1)
|
17
17
|
TIMEZONE = "Asia/Tokyo".freeze
|
18
|
-
|
18
|
+
JOB_START_TIME = 1506407051000
|
19
19
|
DURATIONS = [
|
20
20
|
{from_date: FROM_DATE, to_date: "2015-02-28"}, # It has 7 days between 2015-02-22 and 2015-02-28
|
21
21
|
{from_date: "2015-03-01", to_date: TO_DATE},
|
@@ -45,7 +45,13 @@ module Embulk
|
|
45
45
|
end
|
46
46
|
end
|
47
47
|
end
|
48
|
-
|
48
|
+
def satisfy_task_ignore_start_time(expected_task)
|
49
|
+
satisfy{|input_task|
|
50
|
+
assert_not_nil(input_task[:job_start_time])
|
51
|
+
assert_equal(expected_task, input_task.merge(job_start_time: expected_task[:job_start_time]))
|
52
|
+
true
|
53
|
+
}
|
54
|
+
end
|
49
55
|
def setup_logger
|
50
56
|
stub(Embulk).logger { ::Logger.new(IO::NULL) }
|
51
57
|
end
|
@@ -220,13 +226,12 @@ module Embulk
|
|
220
226
|
|
221
227
|
def test_ignore_early_days
|
222
228
|
stub(Embulk).logger { Logger.new(File::NULL) }
|
223
|
-
|
224
|
-
mock(Mixpanel).resume(task.merge(dates: target_dates), columns, 1, &control)
|
229
|
+
mock(Mixpanel).resume(satisfy_task_ignore_start_time(task.merge(dates: target_dates)), columns, 1, &control)
|
225
230
|
Mixpanel.transaction(transaction_config, &control)
|
226
231
|
end
|
227
232
|
|
228
233
|
def test_info
|
229
|
-
stub(Mixpanel).resume(task.merge(dates: target_dates), columns, 1, &control)
|
234
|
+
stub(Mixpanel).resume(satisfy_task_ignore_start_time(task.merge(dates: target_dates)), columns, 1, &control)
|
230
235
|
|
231
236
|
info_message_regexp = /#{Regexp.escape(target_dates.first)}.+#{Regexp.escape(target_dates.last)}/
|
232
237
|
mock(Embulk.logger).info(info_message_regexp)
|
@@ -236,7 +241,7 @@ module Embulk
|
|
236
241
|
end
|
237
242
|
|
238
243
|
def test_warn
|
239
|
-
stub(Mixpanel).resume(task.merge(dates: target_dates), columns, 1, &control)
|
244
|
+
stub(Mixpanel).resume(satisfy_task_ignore_start_time(task.merge(dates: target_dates)), columns, 1, &control)
|
240
245
|
stub(Embulk.logger).info
|
241
246
|
|
242
247
|
ignore_dates = dates.map{|date| date.to_s}.to_a - target_dates
|
@@ -270,8 +275,7 @@ module Embulk
|
|
270
275
|
class TimezoneTest < self
|
271
276
|
def test_valid_timezone
|
272
277
|
timezone = TIMEZONE
|
273
|
-
mock(Mixpanel).resume(transaction_task(timezone), columns, 1, &control)
|
274
|
-
|
278
|
+
mock(Mixpanel).resume(satisfy_task_ignore_start_time(transaction_task(timezone)), columns, 1, &control)
|
275
279
|
Mixpanel.transaction(transaction_config(timezone), &control)
|
276
280
|
end
|
277
281
|
|
@@ -310,7 +314,7 @@ module Embulk
|
|
310
314
|
def test_valid_days
|
311
315
|
days = 5
|
312
316
|
|
313
|
-
mock(Mixpanel).resume(transaction_task(days), columns, 1, &control)
|
317
|
+
mock(Mixpanel).resume(satisfy_task_ignore_start_time(transaction_task(days)), columns, 1, &control)
|
314
318
|
Mixpanel.transaction(transaction_config(days), &control)
|
315
319
|
end
|
316
320
|
|
@@ -541,7 +545,8 @@ module Embulk
|
|
541
545
|
retry_initial_wait_sec: 0,
|
542
546
|
retry_limit: 3,
|
543
547
|
latest_fetched_time: 0,
|
544
|
-
slice_range: 7
|
548
|
+
slice_range: 7,
|
549
|
+
job_start_time: JOB_START_TIME
|
545
550
|
}
|
546
551
|
end
|
547
552
|
end
|
@@ -571,7 +576,7 @@ module Embulk
|
|
571
576
|
|
572
577
|
def test_run
|
573
578
|
stub(@plugin).preview? { false }
|
574
|
-
mock(@page_builder).add(anything).times(records.length)
|
579
|
+
mock(@page_builder).add(anything).times(records.length * 2)
|
575
580
|
mock(@page_builder).finish
|
576
581
|
|
577
582
|
@plugin.run
|
@@ -580,7 +585,7 @@ module Embulk
|
|
580
585
|
def test_timezone
|
581
586
|
stub(@plugin).preview? { false }
|
582
587
|
adjusted = record_epoch - timezone_offset_seconds
|
583
|
-
mock(@page_builder).add(["FOO", adjusted, "event"]).times(records.length)
|
588
|
+
mock(@page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
|
584
589
|
mock(@page_builder).finish
|
585
590
|
|
586
591
|
@plugin.run
|
@@ -592,9 +597,9 @@ module Embulk
|
|
592
597
|
plugin = Mixpanel.new(task.merge(slice_range: 2), nil, nil, @page_builder)
|
593
598
|
stub(plugin).preview? {false}
|
594
599
|
stub(plugin).fetch(["2015-02-22", "2015-02-23"],0){[]}
|
595
|
-
stub(plugin).fetch(["2015-02-
|
596
|
-
stub(plugin).fetch(["2015-02-
|
597
|
-
stub(plugin).fetch(["2015-02-
|
600
|
+
stub(plugin).fetch(["2015-02-24", "2015-02-25"],0){[]}
|
601
|
+
stub(plugin).fetch(["2015-02-26", "2015-02-27"],0){[]}
|
602
|
+
stub(plugin).fetch(["2015-02-28", "2015-03-01"],0){[]}
|
598
603
|
mock(@page_builder).finish
|
599
604
|
plugin.run
|
600
605
|
end
|
@@ -644,14 +649,8 @@ module Embulk
|
|
644
649
|
mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
|
645
650
|
mock(page_builder).finish
|
646
651
|
any_instance_of(MixpanelApi::Client) do |klass|
|
647
|
-
stub(klass).export(
|
648
|
-
|
649
|
-
}).once do |params, block|
|
650
|
-
records.each{|record| block.call(record) }
|
651
|
-
end
|
652
|
-
stub(klass).export(satisfy{|params|
|
653
|
-
'(abc==def) and properties["mp_processing_time_ms"] > 1234567919'==params["where"]
|
654
|
-
}).once do |params, block|
|
652
|
+
stub(klass).export() do |params, block|
|
653
|
+
assert_equal("(abc==def) and properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME}",params["where"])
|
655
654
|
records.each{|record| block.call(record) }
|
656
655
|
end
|
657
656
|
end
|
@@ -667,14 +666,8 @@ module Embulk
|
|
667
666
|
mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
|
668
667
|
mock(page_builder).finish
|
669
668
|
any_instance_of(MixpanelApi::Client) do |klass|
|
670
|
-
stub(klass).export(
|
671
|
-
|
672
|
-
}).once() do |params,block|
|
673
|
-
records.each{|record| block.call(record) }
|
674
|
-
end
|
675
|
-
stub(klass).export(satisfy {|params|
|
676
|
-
'properties["mp_processing_time_ms"] > 1234567919' == params["where"]
|
677
|
-
}).once() do |params,block|
|
669
|
+
stub(klass).export() do |params, block|
|
670
|
+
assert_equal("properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME}",params["where"])
|
678
671
|
records.each{|record| block.call(record) }
|
679
672
|
end
|
680
673
|
end
|
@@ -782,7 +775,7 @@ module Embulk
|
|
782
775
|
{"int" => properties["int"], "event" => record["event"]}.to_json
|
783
776
|
]
|
784
777
|
|
785
|
-
mock(@page_builder).add(added).times(records.length)
|
778
|
+
mock(@page_builder).add(added).times(records.length * 2)
|
786
779
|
mock(@page_builder).finish
|
787
780
|
|
788
781
|
@plugin.run
|
@@ -834,7 +827,8 @@ module Embulk
|
|
834
827
|
retry_initial_wait_sec: 2,
|
835
828
|
retry_limit: 3,
|
836
829
|
latest_fetched_time: 0,
|
837
|
-
slice_range: 7
|
830
|
+
slice_range: 7,
|
831
|
+
job_start_time: JOB_START_TIME
|
838
832
|
}
|
839
833
|
end
|
840
834
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-mixpanel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yoshihara
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-09-
|
12
|
+
date: 2017-09-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -224,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
224
224
|
version: '0'
|
225
225
|
requirements: []
|
226
226
|
rubyforge_project:
|
227
|
-
rubygems_version: 2.
|
227
|
+
rubygems_version: 2.6.6
|
228
228
|
signing_key:
|
229
229
|
specification_version: 4
|
230
230
|
summary: Mixpanel input plugin for Embulk
|