embulk-input-mixpanel 0.5.3.alpha.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +4 -0
- data/embulk-input-mixpanel.gemspec +1 -1
- data/lib/embulk/input/mixpanel.rb +16 -16
- data/test/embulk/input/test_mixpanel.rb +4 -4
- data/test/prepare_embulk.rb +0 -2
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e5d08021cb9b3e4518546d44eeff78eab3a6e344
|
4
|
+
data.tar.gz: 55b1de01d58db4f50755996917f7f40fdc0fdd33
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 10814321b400f21bfc54acb2005c671265e0fb5a803ab221c7fa5a9dbb02a513a18ea144a1e4dbf9fa92c964b71ca7655f86a2ff866c106df6a5079628b5d3d3
|
7
|
+
data.tar.gz: 85adb7eeac61dec517c2010300e8fa1a58eb0159c86fbe17d0b34ab12209e774fa15ce4bd425f79d495a087519554b46d0eb07627d03d8e601697b2fa212e275
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 0.5.3 - 2017-08-07
|
2
|
+
* [enhancement] Allow user to choose to run incremental or not `incremental` option default to true [#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
|
3
|
+
* [enhancement] Allow user to specify an `incremental_column`, which will be add to the where praramter is API requests[#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
|
4
|
+
* [enhancement] Allow user to specifiy backfill days, this `back_fill_time` option will tell plugin how many days we look back for data [#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
|
5
|
+
|
6
|
+
|
1
7
|
## 0.5.2 - 2017-07-26
|
2
8
|
* [enhancement]Enable realtime data export[#47](https://github.com/treasure-data/embulk-input-mixpanel/pull/47)
|
3
9
|
* [maintenance]Fix incorrect error message[#49](https://github.com/treasure-data/embulk-input-mixpanel/pull/49)
|
data/README.md
CHANGED
@@ -38,6 +38,10 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
|
|
38
38
|
- NOTE: Mixpanel API supports to export data from at least 2 days before to at most the previous day.
|
39
39
|
- **fetch_days**: Count of days range for exporting (integer, optional, default: from_date - (today - 1))
|
40
40
|
- NOTE: Mixpanel doesn't support to from_date > today - 2
|
41
|
+
- **incremental**: Run incremental mode nor not (boolean, optional, default: true)
|
42
|
+
- **incremental_column**: Column to be add to where query as a constraint for incremental time. Only data that have incremental_column timestamp > than previous latest_fetched_time will be return (string, optional, default: nil)
|
43
|
+
- **back_fill_time**: Amount of time that will be subtracted from `from_date` to calculate the final `from_date` that will be use for API Request. This is due to Mixpanel caching data on user devices before sending it to Mixpanel server (integer, optional, default: 5)
|
44
|
+
- NOTE: Only have effect when incremental is true and incremental_column is specified
|
41
45
|
- **fetch_unknown_columns**(deprecated): If you want this plugin fetches unknown (unconfigured in config) columns (boolean, optional, default: false)
|
42
46
|
- NOTE: If true, `unknown_columns` column is created and added unknown columns' data.
|
43
47
|
- **fetch_custom_properties**: All custom properties into `custom_properties` key. "custom properties" are not desribed Mixpanel document [1](https://mixpanel.com/help/questions/articles/special-or-reserved-properties), [2](https://mixpanel.com/help/questions/articles/what-properties-do-mixpanels-libraries-store-by-default). (boolean, optional, default: true)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "embulk-input-mixpanel"
|
3
|
-
spec.version = "0.5.3
|
3
|
+
spec.version = "0.5.3"
|
4
4
|
spec.authors = ["yoshihara", "uu59"]
|
5
5
|
spec.summary = "Mixpanel input plugin for Embulk"
|
6
6
|
spec.description = "Loads records from Mixpanel."
|
@@ -46,9 +46,9 @@ module Embulk
|
|
46
46
|
latest_fetched_time = config.param(:latest_fetched_time, :integer, default: 0)
|
47
47
|
|
48
48
|
# Backfill from date if incremental and an incremental field is set and we are in incremental run
|
49
|
-
if incremental &&
|
49
|
+
if incremental && incremental_column && latest_fetched_time !=0
|
50
50
|
back_fill_days = config.param(:back_fill_days, :integer, default: 5)
|
51
|
-
|
51
|
+
Embulk.logger.info "Backfill days #{back_fill_days}"
|
52
52
|
from_date = (Date.parse(from_date) - back_fill_days).to_s
|
53
53
|
fetch_days = fetch_days.nil? ? nil : fetch_days + back_fill_days
|
54
54
|
end
|
@@ -167,24 +167,24 @@ module Embulk
|
|
167
167
|
Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
|
168
168
|
end
|
169
169
|
record_time_column=@incremental_column || DEFAULT_TIME_COLUMN
|
170
|
-
fetch(dates,prev_latest_fetched_time).each do |record|
|
170
|
+
fetch(dates, prev_latest_fetched_time).each do |record|
|
171
171
|
if @incremental
|
172
172
|
if !record["properties"].include?(record_time_column)
|
173
173
|
raise Embulk::ConfigError.new("Incremental column not exists in fetched data #{record_time_column}")
|
174
174
|
end
|
175
175
|
record_time = record["properties"][record_time_column]
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
176
|
+
if @incremental_column.nil?
|
177
|
+
if record_time <= prev_latest_fetched_time
|
178
|
+
ignored_record_count += 1
|
179
|
+
next
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
current_latest_fetched_time= [
|
184
|
+
current_latest_fetched_time,
|
185
|
+
record_time,
|
186
|
+
].max
|
187
|
+
end
|
188
188
|
values = extract_values(record)
|
189
189
|
if @fetch_unknown_columns
|
190
190
|
unknown_values = extract_unknown_values(record)
|
@@ -272,7 +272,7 @@ module Embulk
|
|
272
272
|
"where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time}"
|
273
273
|
)
|
274
274
|
end
|
275
|
-
|
275
|
+
Embulk.logger.info "Where params is #{params["where"]}"
|
276
276
|
client = MixpanelApi::Client.new(@api_key, @api_secret, self.class.perfect_retry(task))
|
277
277
|
|
278
278
|
if preview?
|
@@ -616,14 +616,14 @@ module Embulk
|
|
616
616
|
|
617
617
|
def test_incremental_column_with_where
|
618
618
|
page_builder = Object.new
|
619
|
-
plugin = Mixpanel.new(task.merge(params: task[:params].merge("where" => "abc==def")), nil, nil, page_builder)
|
619
|
+
plugin = Mixpanel.new(task.merge(params: task[:params].merge("where" => "abc==def"),latest_fetched_time: 1), nil, nil, page_builder)
|
620
620
|
stub(plugin).preview? {false}
|
621
621
|
adjusted = record_epoch - timezone_offset_seconds
|
622
622
|
mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
|
623
623
|
mock(page_builder).finish
|
624
624
|
any_instance_of(MixpanelApi::Client) do |klass|
|
625
625
|
stub(klass).export() do |params, block|
|
626
|
-
assert_equal('(abc==def) and properties["mp_processing_time_ms"] >
|
626
|
+
assert_equal('(abc==def) and properties["mp_processing_time_ms"] > 1',params["where"])
|
627
627
|
records.each{|record| block.call(record) }
|
628
628
|
end
|
629
629
|
end
|
@@ -633,14 +633,14 @@ module Embulk
|
|
633
633
|
|
634
634
|
def test_incremental_column
|
635
635
|
page_builder = Object.new
|
636
|
-
plugin = Mixpanel.new(task, nil, nil, page_builder)
|
636
|
+
plugin = Mixpanel.new(task.merge(latest_fetched_time: 1), nil, nil, page_builder)
|
637
637
|
stub(plugin).preview? {false}
|
638
638
|
adjusted = record_epoch - timezone_offset_seconds
|
639
639
|
mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
|
640
640
|
mock(page_builder).finish
|
641
641
|
any_instance_of(MixpanelApi::Client) do |klass|
|
642
642
|
stub(klass).export() do |params, block|
|
643
|
-
assert_equal('properties["mp_processing_time_ms"] >
|
643
|
+
assert_equal('properties["mp_processing_time_ms"] > 1',params["where"])
|
644
644
|
records.each{|record| block.call(record) }
|
645
645
|
end
|
646
646
|
end
|
data/test/prepare_embulk.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-mixpanel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.3
|
4
|
+
version: 0.5.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yoshihara
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-08-
|
12
|
+
date: 2017-08-07 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -219,12 +219,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
219
219
|
version: '0'
|
220
220
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
221
221
|
requirements:
|
222
|
-
- - "
|
222
|
+
- - ">="
|
223
223
|
- !ruby/object:Gem::Version
|
224
|
-
version:
|
224
|
+
version: '0'
|
225
225
|
requirements: []
|
226
226
|
rubyforge_project:
|
227
|
-
rubygems_version: 2.
|
227
|
+
rubygems_version: 2.4.8
|
228
228
|
signing_key:
|
229
229
|
specification_version: 4
|
230
230
|
summary: Mixpanel input plugin for Embulk
|