embulk-input-mixpanel 0.5.3.alpha.1 → 0.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 28a4798aa8352ee4fbf06a23a57ff44aeb4cb902
4
- data.tar.gz: 433333137ed26c3a2d62b822b767621885095cf4
3
+ metadata.gz: e5d08021cb9b3e4518546d44eeff78eab3a6e344
4
+ data.tar.gz: 55b1de01d58db4f50755996917f7f40fdc0fdd33
5
5
  SHA512:
6
- metadata.gz: 2d5fbba923cf5e52c169ed94001633001a2fcd8bb83cf0138541581d5a7768e4fd51bf7d690f7105c3fb31cf480a27a1af4fa9633636c8e4a237ba5786c99722
7
- data.tar.gz: a147657608530a5f0ef513fdd150e93cf62bee74abeeadc3046ad30df42d19c1b94c4f42606260c827f1785dfc3c5c9fef6cf231efbc55ef4351d7be38cd4002
6
+ metadata.gz: 10814321b400f21bfc54acb2005c671265e0fb5a803ab221c7fa5a9dbb02a513a18ea144a1e4dbf9fa92c964b71ca7655f86a2ff866c106df6a5079628b5d3d3
7
+ data.tar.gz: 85adb7eeac61dec517c2010300e8fa1a58eb0159c86fbe17d0b34ab12209e774fa15ce4bd425f79d495a087519554b46d0eb07627d03d8e601697b2fa212e275
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 0.5.3 - 2017-08-07
2
+ * [enhancement] Allow user to choose to run incremental or not `incremental` option default to true [#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
3
+ * [enhancement] Allow user to specify an `incremental_column`, which will be add to the where praramter is API requests[#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
4
+ * [enhancement] Allow user to specifiy backfill days, this `back_fill_time` option will tell plugin how many days we look back for data [#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
5
+
6
+
1
7
  ## 0.5.2 - 2017-07-26
2
8
  * [enhancement]Enable realtime data export[#47](https://github.com/treasure-data/embulk-input-mixpanel/pull/47)
3
9
  * [maintenance]Fix incorrect error message[#49](https://github.com/treasure-data/embulk-input-mixpanel/pull/49)
data/README.md CHANGED
@@ -38,6 +38,10 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
38
38
  - NOTE: Mixpanel API supports to export data from at least 2 days before to at most the previous day.
39
39
  - **fetch_days**: Count of days range for exporting (integer, optional, default: from_date - (today - 1))
40
40
  - NOTE: Mixpanel doesn't support to from_date > today - 2
41
+ - **incremental**: Run incremental mode nor not (boolean, optional, default: true)
42
+ - **incremental_column**: Column to be add to where query as a constraint for incremental time. Only data that have incremental_column timestamp > than previous latest_fetched_time will be return (string, optional, default: nil)
43
+ - **back_fill_time**: Amount of time that will be subtracted from `from_date` to calculate the final `from_date` that will be use for API Request. This is due to Mixpanel caching data on user devices before sending it to Mixpanel server (integer, optional, default: 5)
44
+ - NOTE: Only have effect when incremental is true and incremental_column is specified
41
45
  - **fetch_unknown_columns**(deprecated): If you want this plugin fetches unknown (unconfigured in config) columns (boolean, optional, default: false)
42
46
  - NOTE: If true, `unknown_columns` column is created and added unknown columns' data.
43
47
  - **fetch_custom_properties**: All custom properties into `custom_properties` key. "custom properties" are not desribed Mixpanel document [1](https://mixpanel.com/help/questions/articles/special-or-reserved-properties), [2](https://mixpanel.com/help/questions/articles/what-properties-do-mixpanels-libraries-store-by-default). (boolean, optional, default: true)
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = "embulk-input-mixpanel"
3
- spec.version = "0.5.3.alpha.1"
3
+ spec.version = "0.5.3"
4
4
  spec.authors = ["yoshihara", "uu59"]
5
5
  spec.summary = "Mixpanel input plugin for Embulk"
6
6
  spec.description = "Loads records from Mixpanel."
@@ -46,9 +46,9 @@ module Embulk
46
46
  latest_fetched_time = config.param(:latest_fetched_time, :integer, default: 0)
47
47
 
48
48
  # Backfill from date if incremental and an incremental field is set and we are in incremental run
49
- if incremental && !incremental_column.nil? && latest_fetched_time !=0
49
+ if incremental && incremental_column && latest_fetched_time !=0
50
50
  back_fill_days = config.param(:back_fill_days, :integer, default: 5)
51
- puts "Backfill days #{back_fill_days}"
51
+ Embulk.logger.info "Backfill days #{back_fill_days}"
52
52
  from_date = (Date.parse(from_date) - back_fill_days).to_s
53
53
  fetch_days = fetch_days.nil? ? nil : fetch_days + back_fill_days
54
54
  end
@@ -167,24 +167,24 @@ module Embulk
167
167
  Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
168
168
  end
169
169
  record_time_column=@incremental_column || DEFAULT_TIME_COLUMN
170
- fetch(dates,prev_latest_fetched_time).each do |record|
170
+ fetch(dates, prev_latest_fetched_time).each do |record|
171
171
  if @incremental
172
172
  if !record["properties"].include?(record_time_column)
173
173
  raise Embulk::ConfigError.new("Incremental column not exists in fetched data #{record_time_column}")
174
174
  end
175
175
  record_time = record["properties"][record_time_column]
176
- if @incremental_column.nil?
177
- if record_time <= prev_latest_fetched_time
178
- ignored_record_count += 1
179
- next
180
- end
181
- end
182
-
183
- current_latest_fetched_time= [
184
- current_latest_fetched_time,
185
- record_time,
186
- ].max
187
- end
176
+ if @incremental_column.nil?
177
+ if record_time <= prev_latest_fetched_time
178
+ ignored_record_count += 1
179
+ next
180
+ end
181
+ end
182
+
183
+ current_latest_fetched_time= [
184
+ current_latest_fetched_time,
185
+ record_time,
186
+ ].max
187
+ end
188
188
  values = extract_values(record)
189
189
  if @fetch_unknown_columns
190
190
  unknown_values = extract_unknown_values(record)
@@ -272,7 +272,7 @@ module Embulk
272
272
  "where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time}"
273
273
  )
274
274
  end
275
- puts "Where params is #{params["where"]}"
275
+ Embulk.logger.info "Where params is #{params["where"]}"
276
276
  client = MixpanelApi::Client.new(@api_key, @api_secret, self.class.perfect_retry(task))
277
277
 
278
278
  if preview?
@@ -616,14 +616,14 @@ module Embulk
616
616
 
617
617
  def test_incremental_column_with_where
618
618
  page_builder = Object.new
619
- plugin = Mixpanel.new(task.merge(params: task[:params].merge("where" => "abc==def")), nil, nil, page_builder)
619
+ plugin = Mixpanel.new(task.merge(params: task[:params].merge("where" => "abc==def"),latest_fetched_time: 1), nil, nil, page_builder)
620
620
  stub(plugin).preview? {false}
621
621
  adjusted = record_epoch - timezone_offset_seconds
622
622
  mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
623
623
  mock(page_builder).finish
624
624
  any_instance_of(MixpanelApi::Client) do |klass|
625
625
  stub(klass).export() do |params, block|
626
- assert_equal('(abc==def) and properties["mp_processing_time_ms"] > 0',params["where"])
626
+ assert_equal('(abc==def) and properties["mp_processing_time_ms"] > 1',params["where"])
627
627
  records.each{|record| block.call(record) }
628
628
  end
629
629
  end
@@ -633,14 +633,14 @@ module Embulk
633
633
 
634
634
  def test_incremental_column
635
635
  page_builder = Object.new
636
- plugin = Mixpanel.new(task, nil, nil, page_builder)
636
+ plugin = Mixpanel.new(task.merge(latest_fetched_time: 1), nil, nil, page_builder)
637
637
  stub(plugin).preview? {false}
638
638
  adjusted = record_epoch - timezone_offset_seconds
639
639
  mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
640
640
  mock(page_builder).finish
641
641
  any_instance_of(MixpanelApi::Client) do |klass|
642
642
  stub(klass).export() do |params, block|
643
- assert_equal('properties["mp_processing_time_ms"] > 0',params["where"])
643
+ assert_equal('properties["mp_processing_time_ms"] > 1',params["where"])
644
644
  records.each{|record| block.call(record) }
645
645
  end
646
646
  end
@@ -1,6 +1,4 @@
1
1
  module PrepareEmbulk
2
- require "embulk/command/embulk_run"
3
-
4
2
  if Embulk.respond_to?(:home)
5
3
  # keep compatibility for Embulk 0.6.x
6
4
  classpath_dir = Embulk.home("classpath")
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-mixpanel
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.3.alpha.1
4
+ version: 0.5.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - yoshihara
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-08-02 00:00:00.000000000 Z
12
+ date: 2017-08-07 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -219,12 +219,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
219
219
  version: '0'
220
220
  required_rubygems_version: !ruby/object:Gem::Requirement
221
221
  requirements:
222
- - - ">"
222
+ - - ">="
223
223
  - !ruby/object:Gem::Version
224
- version: 1.3.1
224
+ version: '0'
225
225
  requirements: []
226
226
  rubyforge_project:
227
- rubygems_version: 2.6.12
227
+ rubygems_version: 2.4.8
228
228
  signing_key:
229
229
  specification_version: 4
230
230
  summary: Mixpanel input plugin for Embulk