embulk-input-mixpanel 0.5.8 → 0.5.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +6 -2
  3. data/embulk-input-mixpanel.gemspec +1 -1
  4. data/lib/embulk/input/mixpanel.rb +11 -2
  5. data/test/embulk/input/test_mixpanel.rb +12 -5
  6. metadata +35 -35
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 918d88c52e1dd983b7d78dfa673ce5f946ef0629
4
- data.tar.gz: 9d17d52d3c3daf84f3b11d6b068ff2b873627ab7
3
+ metadata.gz: 71b5e68b20fb92e74d3a3210ee1700f90bdf26bc
4
+ data.tar.gz: b29ffe1fdf6f9f759cb6d0790e1448da6fd34ec2
5
5
  SHA512:
6
- metadata.gz: 235b268d682ff9a4a964d37455ea873cd331b78bd9a2d44e7053b584e7910d5da4f5a2087c9ce9497d30fc06672e1956ce94f8a64885bdbbbd38c0bf7cfbca97
7
- data.tar.gz: bb8c5c8808e95b636c41ae5f72f0f8135390face7765b3fbb6d7f3940da95dbaf6e6f034541597b926abfc2f63efa56eef7c14069d36015dbc90ec490a646e34
6
+ metadata.gz: 527ebe18c1cf8a7e6f8ab553f4b272f1719c09c8df4f2a3530f2ba267b7d633056cb6641b9681117f4fb2497a1cf4265ede7cce75b5f6e947e24b5333b55fd5d
7
+ data.tar.gz: 279e5750a346ed4b04c104418d56b7b7e7427d5a580ad1f7481e124b5ba893d4c370eb27577686db9fa11f6d1f18d513f22504441353a4547082356163a5d1be
data/CHANGELOG.md CHANGED
@@ -1,7 +1,11 @@
1
+ ## 0.5.9 - 2017-11-10
2
+
3
+ * [enhancement] Add upper limit delay to incremental column query [#56](https://github.com/treasure-data/embulk-input-mixpanel/pull/56)
4
+
1
5
  ## 0.5.8 - 2017-09-26
2
6
 
3
- * [bug] Fix issue when back_fill data get processed by Mixpanel when plugin is running [#56](https://github.com/treasure-data/embulk-input-mixpanel/pull/56)
4
- * [bug] Revert PR #54 [#56](https://github.com/treasure-data/embulk-input-mixpanel/pull/56)
7
+ * [bug] Fix issue when back_fill data get processed by Mixpanel when plugin is running
8
+ * [bug] Revert PR #54
5
9
 
6
10
  ## 0.5.6 - 2017-09-20
7
11
  * [bug] Fix issue when back_fill data get processed by Mixpanel when plugin is running [#54](https://github.com/treasure-data/embulk-input-mixpanel/pull/54)
data/embulk-input-mixpanel.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = "embulk-input-mixpanel"
3
- spec.version = "0.5.8"
3
+ spec.version = "0.5.9"
4
4
  spec.authors = ["yoshihara", "uu59"]
5
5
  spec.summary = "Mixpanel input plugin for Embulk"
6
6
  spec.description = "Loads records from Mixpanel."
data/lib/embulk/input/mixpanel.rb CHANGED
@@ -52,6 +52,9 @@ module Embulk
52
52
 
53
53
  range = RangeGenerator.new(from_date, fetch_days).generate_range
54
54
  Embulk.logger.info "Try to fetch data from #{range.first} to #{range.last}"
55
+ job_start_time = Time.now.to_i*1000
56
+ upper_limit_delay = config.param(:incremental_column_upper_limit_delay_in_seconds, :integer, default: 0)
57
+ incremental_column_upper_limit = job_start_time - (upper_limit_delay * 1000)
55
58
  task = {
56
59
  params: export_params(config),
57
60
  dates: range,
@@ -67,9 +70,14 @@ module Embulk
67
70
  latest_fetched_time: latest_fetched_time,
68
71
  incremental: incremental,
69
72
  slice_range: config.param(:slice_range, :integer, default: 7),
70
- job_start_time: Time.now.to_i*1000
73
+ job_start_time: job_start_time,
74
+ incremental_column_upper_limit: incremental_column_upper_limit
71
75
  }
72
76
 
77
+ if !incremental_column.nil? && !latest_fetched_time.nil? && (incremental_column_upper_limit <= latest_fetched_time)
78
+ raise Embulk::ConfigError.new("Incremental column upper limit (job_start_time - incremental_column_upper_limit_delay_in_seconds) can't be smaller or equal latest fetched time #{latest_fetched_time}")
79
+ end
80
+
73
81
  if task[:fetch_unknown_columns] && task[:fetch_custom_properties]
74
82
  raise Embulk::ConfigError.new("Don't set true both `fetch_unknown_columns` and `fetch_custom_properties`.")
75
83
  end
@@ -156,6 +164,7 @@ module Embulk
156
164
  end
157
165
 
158
166
  def run
167
+ Embulk.logger.info "Job start time is #{task[:job_start_time]}"
159
168
  self.class.giveup_when_mixpanel_is_down
160
169
  prev_latest_fetched_time = task[:latest_fetched_time] || 0
161
170
  prev_latest_fetched_time_format = Time.at(prev_latest_fetched_time).strftime("%F %T %z")
@@ -267,7 +276,7 @@ module Embulk
267
276
  )
268
277
  if !@incremental_column.nil? # can't do filter on time column, time column need to be filter manually.
269
278
  params = params.merge(
270
- "where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time || 0} and properties[\"#{@incremental_column}\"] < #{task[:job_start_time]}"
279
+ "where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time || 0} and properties[\"#{@incremental_column}\"] < #{task[:incremental_column_upper_limit]}"
271
280
  )
272
281
  end
273
282
  Embulk.logger.info "Where params is #{params["where"]}"
data/test/embulk/input/test_mixpanel.rb CHANGED
@@ -16,6 +16,7 @@ module Embulk
16
16
  DATES = Date.parse(FROM_DATE)..(Date.parse(FROM_DATE) + DAYS - 1)
17
17
  TIMEZONE = "Asia/Tokyo".freeze
18
18
  JOB_START_TIME = 1506407051000
19
+ UPPER_LIMIT_DELAY = 300 # 300 seconds delay
19
20
  DURATIONS = [
20
21
  {from_date: FROM_DATE, to_date: "2015-02-28"}, # It has 7 days between 2015-02-22 and 2015-02-28
21
22
  {from_date: "2015-03-01", to_date: TO_DATE},
@@ -46,9 +47,13 @@ module Embulk
46
47
  end
47
48
  end
48
49
  def satisfy_task_ignore_start_time(expected_task)
49
- satisfy{|input_task|
50
+ satisfy {|input_task|
50
51
  assert_not_nil(input_task[:job_start_time])
51
- assert_equal(expected_task, input_task.merge(job_start_time: expected_task[:job_start_time]))
52
+ # This will also verify incremental column upper limit
53
+ assert_equal(input_task[:incremental_column_upper_limit], input_task[:job_start_time] - UPPER_LIMIT_DELAY * 1000)
54
+ expected_task[:job_start_time] = input_task[:job_start_time]
55
+ expected_task[:incremental_column_upper_limit] = input_task[:incremental_column_upper_limit]
56
+ assert_equal(expected_task, input_task)
52
57
  true
53
58
  }
54
59
  end
@@ -650,7 +655,7 @@ module Embulk
650
655
  mock(page_builder).finish
651
656
  any_instance_of(MixpanelApi::Client) do |klass|
652
657
  stub(klass).export() do |params, block|
653
- assert_equal("(abc==def) and properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME}",params["where"])
658
+ assert_equal("(abc==def) and properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME - UPPER_LIMIT_DELAY * 1000}",params["where"])
654
659
  records.each{|record| block.call(record) }
655
660
  end
656
661
  end
@@ -667,7 +672,7 @@ module Embulk
667
672
  mock(page_builder).finish
668
673
  any_instance_of(MixpanelApi::Client) do |klass|
669
674
  stub(klass).export() do |params, block|
670
- assert_equal("properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME}",params["where"])
675
+ assert_equal("properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME - UPPER_LIMIT_DELAY * 1000}",params["where"])
671
676
  records.each{|record| block.call(record) }
672
677
  end
673
678
  end
@@ -828,7 +833,8 @@ module Embulk
828
833
  retry_limit: 3,
829
834
  latest_fetched_time: 0,
830
835
  slice_range: 7,
831
- job_start_time: JOB_START_TIME
836
+ job_start_time: JOB_START_TIME,
837
+ incremental_column_upper_limit: (JOB_START_TIME - UPPER_LIMIT_DELAY * 1000)
832
838
  }
833
839
  end
834
840
 
@@ -865,6 +871,7 @@ module Embulk
865
871
  retry_initial_wait_sec: 2,
866
872
  retry_limit: 3,
867
873
  latest_fetched_time: 0,
874
+ incremental_column_upper_limit_delay_in_seconds: UPPER_LIMIT_DELAY
868
875
  }
869
876
  end
870
877
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-mixpanel
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.8
4
+ version: 0.5.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - yoshihara
@@ -9,79 +9,80 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-09-26 00:00:00.000000000 Z
12
+ date: 2017-11-10 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
+ name: httpclient
15
16
  requirement: !ruby/object:Gem::Requirement
16
17
  requirements:
17
18
  - - ">="
18
19
  - !ruby/object:Gem::Version
19
20
  version: 2.8.3
20
- name: httpclient
21
- prerelease: false
22
- type: :runtime
23
21
  version_requirements: !ruby/object:Gem::Requirement
24
22
  requirements:
25
23
  - - ">="
26
24
  - !ruby/object:Gem::Version
27
25
  version: 2.8.3
26
+ prerelease: false
27
+ type: :runtime
28
28
  - !ruby/object:Gem::Dependency
29
+ name: tzinfo
29
30
  requirement: !ruby/object:Gem::Requirement
30
31
  requirements:
31
32
  - - ">="
32
33
  - !ruby/object:Gem::Version
33
34
  version: '0'
34
- name: tzinfo
35
- prerelease: false
36
- type: :runtime
37
35
  version_requirements: !ruby/object:Gem::Requirement
38
36
  requirements:
39
37
  - - ">="
40
38
  - !ruby/object:Gem::Version
41
39
  version: '0'
40
+ prerelease: false
41
+ type: :runtime
42
42
  - !ruby/object:Gem::Dependency
43
+ name: perfect_retry
43
44
  requirement: !ruby/object:Gem::Requirement
44
45
  requirements:
45
46
  - - "~>"
46
47
  - !ruby/object:Gem::Version
47
48
  version: '0.5'
48
- name: perfect_retry
49
- prerelease: false
50
- type: :runtime
51
49
  version_requirements: !ruby/object:Gem::Requirement
52
50
  requirements:
53
51
  - - "~>"
54
52
  - !ruby/object:Gem::Version
55
53
  version: '0.5'
54
+ prerelease: false
55
+ type: :runtime
56
56
  - !ruby/object:Gem::Dependency
57
+ name: bundler
57
58
  requirement: !ruby/object:Gem::Requirement
58
59
  requirements:
59
60
  - - "~>"
60
61
  - !ruby/object:Gem::Version
61
62
  version: '1.0'
62
- name: bundler
63
- prerelease: false
64
- type: :development
65
63
  version_requirements: !ruby/object:Gem::Requirement
66
64
  requirements:
67
65
  - - "~>"
68
66
  - !ruby/object:Gem::Version
69
67
  version: '1.0'
68
+ prerelease: false
69
+ type: :development
70
70
  - !ruby/object:Gem::Dependency
71
+ name: rake
71
72
  requirement: !ruby/object:Gem::Requirement
72
73
  requirements:
73
74
  - - ">="
74
75
  - !ruby/object:Gem::Version
75
76
  version: '10.0'
76
- name: rake
77
- prerelease: false
78
- type: :development
79
77
  version_requirements: !ruby/object:Gem::Requirement
80
78
  requirements:
81
79
  - - ">="
82
80
  - !ruby/object:Gem::Version
83
81
  version: '10.0'
82
+ prerelease: false
83
+ type: :development
84
84
  - !ruby/object:Gem::Dependency
85
+ name: embulk
85
86
  requirement: !ruby/object:Gem::Requirement
86
87
  requirements:
87
88
  - - ">="
@@ -90,9 +91,6 @@ dependencies:
90
91
  - - "<"
91
92
  - !ruby/object:Gem::Version
92
93
  version: '1.0'
93
- name: embulk
94
- prerelease: false
95
- type: :development
96
94
  version_requirements: !ruby/object:Gem::Requirement
97
95
  requirements:
98
96
  - - ">="
@@ -101,76 +99,78 @@ dependencies:
101
99
  - - "<"
102
100
  - !ruby/object:Gem::Version
103
101
  version: '1.0'
102
+ prerelease: false
103
+ type: :development
104
104
  - !ruby/object:Gem::Dependency
105
+ name: pry
105
106
  requirement: !ruby/object:Gem::Requirement
106
107
  requirements:
107
108
  - - ">="
108
109
  - !ruby/object:Gem::Version
109
110
  version: '0'
110
- name: pry
111
- prerelease: false
112
- type: :development
113
111
  version_requirements: !ruby/object:Gem::Requirement
114
112
  requirements:
115
113
  - - ">="
116
114
  - !ruby/object:Gem::Version
117
115
  version: '0'
116
+ prerelease: false
117
+ type: :development
118
118
  - !ruby/object:Gem::Dependency
119
+ name: test-unit
119
120
  requirement: !ruby/object:Gem::Requirement
120
121
  requirements:
121
122
  - - ">="
122
123
  - !ruby/object:Gem::Version
123
124
  version: '0'
124
- name: test-unit
125
- prerelease: false
126
- type: :development
127
125
  version_requirements: !ruby/object:Gem::Requirement
128
126
  requirements:
129
127
  - - ">="
130
128
  - !ruby/object:Gem::Version
131
129
  version: '0'
130
+ prerelease: false
131
+ type: :development
132
132
  - !ruby/object:Gem::Dependency
133
+ name: test-unit-rr
133
134
  requirement: !ruby/object:Gem::Requirement
134
135
  requirements:
135
136
  - - ">="
136
137
  - !ruby/object:Gem::Version
137
138
  version: '0'
138
- name: test-unit-rr
139
- prerelease: false
140
- type: :development
141
139
  version_requirements: !ruby/object:Gem::Requirement
142
140
  requirements:
143
141
  - - ">="
144
142
  - !ruby/object:Gem::Version
145
143
  version: '0'
144
+ prerelease: false
145
+ type: :development
146
146
  - !ruby/object:Gem::Dependency
147
+ name: codeclimate-test-reporter
147
148
  requirement: !ruby/object:Gem::Requirement
148
149
  requirements:
149
150
  - - "~>"
150
151
  - !ruby/object:Gem::Version
151
152
  version: '0.5'
152
- name: codeclimate-test-reporter
153
- prerelease: false
154
- type: :development
155
153
  version_requirements: !ruby/object:Gem::Requirement
156
154
  requirements:
157
155
  - - "~>"
158
156
  - !ruby/object:Gem::Version
159
157
  version: '0.5'
158
+ prerelease: false
159
+ type: :development
160
160
  - !ruby/object:Gem::Dependency
161
+ name: everyleaf-embulk_helper
161
162
  requirement: !ruby/object:Gem::Requirement
162
163
  requirements:
163
164
  - - ">="
164
165
  - !ruby/object:Gem::Version
165
166
  version: '0'
166
- name: everyleaf-embulk_helper
167
- prerelease: false
168
- type: :development
169
167
  version_requirements: !ruby/object:Gem::Requirement
170
168
  requirements:
171
169
  - - ">="
172
170
  - !ruby/object:Gem::Version
173
171
  version: '0'
172
+ prerelease: false
173
+ type: :development
174
174
  description: Loads records from Mixpanel.
175
175
  email:
176
176
  - h.yoshihara@everyleaf.com