embulk-input-mixpanel 0.5.3.alpha.1 → 0.5.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +4 -0
- data/embulk-input-mixpanel.gemspec +1 -1
- data/lib/embulk/input/mixpanel.rb +16 -16
- data/test/embulk/input/test_mixpanel.rb +4 -4
- data/test/prepare_embulk.rb +0 -2
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e5d08021cb9b3e4518546d44eeff78eab3a6e344
|
4
|
+
data.tar.gz: 55b1de01d58db4f50755996917f7f40fdc0fdd33
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 10814321b400f21bfc54acb2005c671265e0fb5a803ab221c7fa5a9dbb02a513a18ea144a1e4dbf9fa92c964b71ca7655f86a2ff866c106df6a5079628b5d3d3
|
7
|
+
data.tar.gz: 85adb7eeac61dec517c2010300e8fa1a58eb0159c86fbe17d0b34ab12209e774fa15ce4bd425f79d495a087519554b46d0eb07627d03d8e601697b2fa212e275
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 0.5.3 - 2017-08-07
|
2
|
+
* [enhancement] Allow user to choose to run incremental or not `incremental` option default to true [#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
|
3
|
+
* [enhancement] Allow user to specify an `incremental_column`, which will be add to the where praramter is API requests[#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
|
4
|
+
* [enhancement] Allow user to specifiy backfill days, this `back_fill_time` option will tell plugin how many days we look back for data [#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
|
5
|
+
|
6
|
+
|
1
7
|
## 0.5.2 - 2017-07-26
|
2
8
|
* [enhancement]Enable realtime data export[#47](https://github.com/treasure-data/embulk-input-mixpanel/pull/47)
|
3
9
|
* [maintenance]Fix incorrect error message[#49](https://github.com/treasure-data/embulk-input-mixpanel/pull/49)
|
data/README.md
CHANGED
@@ -38,6 +38,10 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
|
|
38
38
|
- NOTE: Mixpanel API supports to export data from at least 2 days before to at most the previous day.
|
39
39
|
- **fetch_days**: Count of days range for exporting (integer, optional, default: from_date - (today - 1))
|
40
40
|
- NOTE: Mixpanel doesn't support to from_date > today - 2
|
41
|
+
- **incremental**: Run incremental mode nor not (boolean, optional, default: true)
|
42
|
+
- **incremental_column**: Column to be add to where query as a constraint for incremental time. Only data that have incremental_column timestamp > than previous latest_fetched_time will be return (string, optional, default: nil)
|
43
|
+
- **back_fill_time**: Amount of time that will be subtracted from `from_date` to calculate the final `from_date` that will be use for API Request. This is due to Mixpanel caching data on user devices before sending it to Mixpanel server (integer, optional, default: 5)
|
44
|
+
- NOTE: Only have effect when incremental is true and incremental_column is specified
|
41
45
|
- **fetch_unknown_columns**(deprecated): If you want this plugin fetches unknown (unconfigured in config) columns (boolean, optional, default: false)
|
42
46
|
- NOTE: If true, `unknown_columns` column is created and added unknown columns' data.
|
43
47
|
- **fetch_custom_properties**: All custom properties into `custom_properties` key. "custom properties" are not desribed Mixpanel document [1](https://mixpanel.com/help/questions/articles/special-or-reserved-properties), [2](https://mixpanel.com/help/questions/articles/what-properties-do-mixpanels-libraries-store-by-default). (boolean, optional, default: true)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "embulk-input-mixpanel"
|
3
|
-
spec.version = "0.5.3
|
3
|
+
spec.version = "0.5.3"
|
4
4
|
spec.authors = ["yoshihara", "uu59"]
|
5
5
|
spec.summary = "Mixpanel input plugin for Embulk"
|
6
6
|
spec.description = "Loads records from Mixpanel."
|
@@ -46,9 +46,9 @@ module Embulk
|
|
46
46
|
latest_fetched_time = config.param(:latest_fetched_time, :integer, default: 0)
|
47
47
|
|
48
48
|
# Backfill from date if incremental and an incremental field is set and we are in incremental run
|
49
|
-
if incremental &&
|
49
|
+
if incremental && incremental_column && latest_fetched_time !=0
|
50
50
|
back_fill_days = config.param(:back_fill_days, :integer, default: 5)
|
51
|
-
|
51
|
+
Embulk.logger.info "Backfill days #{back_fill_days}"
|
52
52
|
from_date = (Date.parse(from_date) - back_fill_days).to_s
|
53
53
|
fetch_days = fetch_days.nil? ? nil : fetch_days + back_fill_days
|
54
54
|
end
|
@@ -167,24 +167,24 @@ module Embulk
|
|
167
167
|
Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
|
168
168
|
end
|
169
169
|
record_time_column=@incremental_column || DEFAULT_TIME_COLUMN
|
170
|
-
fetch(dates,prev_latest_fetched_time).each do |record|
|
170
|
+
fetch(dates, prev_latest_fetched_time).each do |record|
|
171
171
|
if @incremental
|
172
172
|
if !record["properties"].include?(record_time_column)
|
173
173
|
raise Embulk::ConfigError.new("Incremental column not exists in fetched data #{record_time_column}")
|
174
174
|
end
|
175
175
|
record_time = record["properties"][record_time_column]
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
176
|
+
if @incremental_column.nil?
|
177
|
+
if record_time <= prev_latest_fetched_time
|
178
|
+
ignored_record_count += 1
|
179
|
+
next
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
current_latest_fetched_time= [
|
184
|
+
current_latest_fetched_time,
|
185
|
+
record_time,
|
186
|
+
].max
|
187
|
+
end
|
188
188
|
values = extract_values(record)
|
189
189
|
if @fetch_unknown_columns
|
190
190
|
unknown_values = extract_unknown_values(record)
|
@@ -272,7 +272,7 @@ module Embulk
|
|
272
272
|
"where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time}"
|
273
273
|
)
|
274
274
|
end
|
275
|
-
|
275
|
+
Embulk.logger.info "Where params is #{params["where"]}"
|
276
276
|
client = MixpanelApi::Client.new(@api_key, @api_secret, self.class.perfect_retry(task))
|
277
277
|
|
278
278
|
if preview?
|
@@ -616,14 +616,14 @@ module Embulk
|
|
616
616
|
|
617
617
|
def test_incremental_column_with_where
|
618
618
|
page_builder = Object.new
|
619
|
-
plugin = Mixpanel.new(task.merge(params: task[:params].merge("where" => "abc==def")), nil, nil, page_builder)
|
619
|
+
plugin = Mixpanel.new(task.merge(params: task[:params].merge("where" => "abc==def"),latest_fetched_time: 1), nil, nil, page_builder)
|
620
620
|
stub(plugin).preview? {false}
|
621
621
|
adjusted = record_epoch - timezone_offset_seconds
|
622
622
|
mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
|
623
623
|
mock(page_builder).finish
|
624
624
|
any_instance_of(MixpanelApi::Client) do |klass|
|
625
625
|
stub(klass).export() do |params, block|
|
626
|
-
assert_equal('(abc==def) and properties["mp_processing_time_ms"] >
|
626
|
+
assert_equal('(abc==def) and properties["mp_processing_time_ms"] > 1',params["where"])
|
627
627
|
records.each{|record| block.call(record) }
|
628
628
|
end
|
629
629
|
end
|
@@ -633,14 +633,14 @@ module Embulk
|
|
633
633
|
|
634
634
|
def test_incremental_column
|
635
635
|
page_builder = Object.new
|
636
|
-
plugin = Mixpanel.new(task, nil, nil, page_builder)
|
636
|
+
plugin = Mixpanel.new(task.merge(latest_fetched_time: 1), nil, nil, page_builder)
|
637
637
|
stub(plugin).preview? {false}
|
638
638
|
adjusted = record_epoch - timezone_offset_seconds
|
639
639
|
mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
|
640
640
|
mock(page_builder).finish
|
641
641
|
any_instance_of(MixpanelApi::Client) do |klass|
|
642
642
|
stub(klass).export() do |params, block|
|
643
|
-
assert_equal('properties["mp_processing_time_ms"] >
|
643
|
+
assert_equal('properties["mp_processing_time_ms"] > 1',params["where"])
|
644
644
|
records.each{|record| block.call(record) }
|
645
645
|
end
|
646
646
|
end
|
data/test/prepare_embulk.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-mixpanel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.3
|
4
|
+
version: 0.5.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yoshihara
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-08-
|
12
|
+
date: 2017-08-07 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -219,12 +219,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
219
219
|
version: '0'
|
220
220
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
221
221
|
requirements:
|
222
|
-
- - "
|
222
|
+
- - ">="
|
223
223
|
- !ruby/object:Gem::Version
|
224
|
-
version:
|
224
|
+
version: '0'
|
225
225
|
requirements: []
|
226
226
|
rubyforge_project:
|
227
|
-
rubygems_version: 2.
|
227
|
+
rubygems_version: 2.4.8
|
228
228
|
signing_key:
|
229
229
|
specification_version: 4
|
230
230
|
summary: Mixpanel input plugin for Embulk
|