embulk-input-mixpanel 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d553d9b694f3b3150c2c634f1f4b334e7c2162e9
+  data.tar.gz: 89480ccc7dc13ace59cb88769af0263b0c078c72
+SHA512:
+  metadata.gz: 60063447a33ad85b84d33bb01d85ad757c3bcc0494908f30a9c8c78e8fb5df17049c27833b1ce6acccf52358b6575b036ebdbd770fbff4e38558ecdd90734120
+  data.tar.gz: d98177f66557bec6a845749bf93ac69a4b4b8bbbbf63695222f945d37b1d6832432eb58c8448b354018c819e676d5fcf48409b723a32fd488291fd0b724f6bd6

data/.gitignore

@@ -0,0 +1,5 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/Gemfile.lock

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+rvm:
+- jruby-19mode
+addons:
+  code_climate:
+    repo_token:
+      secure: "opE/ZhRzsEU2Fn6YEnItMD/rMs3O2OHVYQQ7Ly0dAkIc9ZrVMa//ogt4k7h0QGZUettRVV7kawCtRzde+QbOmezTRwWqQJ7Mi6D6qfwlYMz+D9FcufHaCJyLy+dYeBuQqr1d6n/3gVqVGh8MAMGHipYodudCub38DQ1sVWcCMDNNo4PMRFLS0pS839SC80HAS7tutOeaeohRc+Ct5yoY2ZDoTuGtNeJhwqmDAo13RwvVu9aZw97EZPvt8UcaW5oYDnx47kDpKi4XGrTPaWpSm/IitwW11FF+Kevt0RpUS0uVWqg4/6xTjDg++ETG+94ePYrOZGF4ne/CPtC0AtaWF1jgSlL9iu4IR/Awt+2BqawKzCnoSRgCGIHRZApErC3KacadJBaPCrKwf4xxxqRXex4lXcptKRygkG4ic2r+MblyPPwIsc1Wb1QYCeVjIEGWzOgKVEUpT8qN0DAj6KQe/HuyuXKE/FiPIRfkJkRY5oGCNZtzeCzXlC+IqhZYCg6HwcZuk3CbcxRrg5VFKDlL0VWacD/FQoGgfTp3SUmaL6NMcCKSrL0vjEgx98/yM9rsGZlZOU9ioN93PQVgqpI5dl+nm9vKkFyuzW07nNM+/6PNdHbbZBY1OLNd3RSpWVC9TDtZ3Q2gAx25+31TDzD2/3yjJjfdtwIH3bdyrO44MK8="
+notifications:
+  slack:
+    secure: cAAHcb/HdJK6elr6gfxlotUJdZys8nA/BUb1rKVt7T7EShOjqNOwHo0vZBG5fXQEbUJjHCFAl8tH1N8wz+KLoVErlMmD2OgBKGgvgm3fddkIgGRRvxe8yB0yEFy5yuvfgqJDlwPHZxbwj2Ddb8uTaLd8yzmRzGWNwLyRQTDaQbaKBaB+emP2t/1AilQHyItN02NuW3qcEGr7Hzg+IsKabDRzhMi89RV0xk2eP5iWTYW76ULbbfP3NqBpcVLskAqVQd1YOFUSEb2fdzRj7UuLUUp3C0Kb5gKfXPHb97XuMqwyviakTr02omdaM82NvB8GKn7QKBbcwA1Iz1euinxQQ7MTzJe8wLMH8q205lTpl96CANKwIxrk/HestgEboq97xSp5Wc1Qn1ynnrLOvBYW7EG/e6exguojQmkLkV9P6LW9g4nYsR8UEjyuRd252xch4+b4M3lxtlAXtpWguPfm5ZKhICBsJMlYm6//q1cBlNUOzR0cufCHDTNdlB/YLynvdg1ykFISYARP+5sV2YM9rD/1Rs/396qYr0mNbCh83P9GlZ5NZtWLQePJroHe3iQ18X3YHEcb50OTfRmKNoXTldQBD9iKNkwlFpWfruVdBWT+JRM77+yIV9ORjY7UE4ZtH8HxAg4bX+ey+C5axgxQk569h3CdLmVapFt6wovCSVk=
+jdk:
+- oraclejdk8
+gemfile:
+  - gemfiles/embulk-0.6.16
+  - gemfiles/embulk-0.6.17
+  - gemfiles/embulk-latest

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 - 2015-07-28
+
+The first release!!

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2015 Everyleaf Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,53 @@
+[![Build Status](https://travis-ci.org/treasure-data/embulk-input-mixpanel.svg?branch=master)](https://travis-ci.org/treasure-data/embulk-input-mixpanel)
+[![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-mixpanel/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-mixpanel)
+[![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-mixpanel/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-mixpanel/coverage)
+
+# Mixpanel input plugin for Embulk
+
+embulk-input-mixpanel is the Embulk input plugin for [Mixpanel](https://mixpanel.com).
+
+## Overview
+
+Required Embulk version >= 0.6.16.
+
+* **Plugin type**: input
+* **Resume supported**: no
+* **Cleanup supported**: no
+* **Guess supported**: yes
+
+## Setup
+
+### How to get API configuration
+
+This plugin uses API key and API secret for target project. Before you make your config.yml, you should get API key and API secret in mixpanel website.
+
+For API configuration, you should log in mixpanel website, and click "Account" at the header. When you select "Projects" panel, you can get "API Key" and "API Secret" for each project.
+
+### Configuration
+
+- **api_key**: project API Key (string, required)
+- **api_secret**: project API Secret (string, required)
+- **timezone**: project timezone(string, required)
+- **from_date**: From date to export (string, required)
+- **to_date**: To date to export (string, required)
+- **event**: The event or events to filter data (array, optional, default: nil)
+- **where**: Expression to filter data (c.f. https://mixpanel.com/docs/api-documentation/data-export-api#segmentation-expressions) (string, optional, default: nil)
+- **bucket**:The data backet to filter data (string, optional, default: nil)
+
+## Example
+
+```yaml
+in:
+  type: mixpanel
+  api_key: "API_KEY"
+  api_secret: "API_SECRET"
+  timezone: "US/Pacific"
+  from_date: "2015-07-19"
+  to_date: "2015-07-20"
+```
+
+## Run test
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,60 @@
+require "bundler/gem_tasks"
+
+task default: :test
+
+desc "Run tests"
+task :test do
+  ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
+end
+
+namespace :release do
+  desc "Add header of now version release to ChangeLog and bump up version"
+  task :prepare do
+    root_dir = Pathname.new(File.expand_path("../", __FILE__))
+    changelog_file = root_dir.join("CHANGELOG.md")
+    gemspec_file = root_dir.join("embulk-input-mixpanel.gemspec")
+
+    system("git fetch origin")
+
+    # detect merged PR
+    old_version = gemspec_file.read[/spec\.version += *"([0-9]+\.[0-9]+\.[0-9]+)"/, 1]
+    pr_numbers = `git log v#{old_version}..origin/master --oneline`.scan(/#[0-9]+/)
+
+    if !$?.success? || pr_numbers.empty?
+      puts "Detecting PR failed. Please confirm if any PR were merged after the latest release."
+      exit(false)
+    end
+
+    # Generate new version
+    major, minor, patch = old_version.split(".").map(&:to_i)
+    new_version = "#{major}.#{minor}.#{patch + 1}"
+
+    # Update ChangeLog
+    pr_descriptions = pr_numbers.map do |number|
+      body = open("https://api.github.com/repos/treasure-data/embulk-input-mixpanel/issues/#{number.gsub("#", "")}").read
+      payload = JSON.parse(body)
+      "* [] #{payload["title"]} [#{number}](https://github.com/treasure-data/embulk-input-mixpanel/pull/#{number.gsub('#', '')})"
+    end.join("\n")
+
+    new_changelog = <<-HEADER
+## #{new_version} - #{Time.now.strftime("%Y-%m-%d")}
+#{pr_descriptions}
+
+#{changelog_file.read.chomp}
+HEADER
+
+    File.open(changelog_file, "w") {|f| f.write(new_changelog) }
+
+    # Update version.rb
+    old_content = gemspec_file.read
+    File.open(gemspec_file, "w") do |f|
+      f.write old_content.gsub(/(spec\.version += *)".*?"/, %Q!\\1"#{new_version}"!)
+    end
+
+    # Update Gemfile.lock
+    system("bundle install")
+
+    puts "ChangeLog, version and Gemfile.lock were updated. New version is #{new_version}."
+  end
+end
+

data/embulk-input-mixpanel.gemspec

@@ -0,0 +1,25 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-mixpanel"
+  spec.version       = "0.1.0"
+  spec.authors       = ["yoshihara", "uu59"]
+  spec.summary       = "Mixpanel input plugin for Embulk"
+  spec.description   = "Loads records from Mixpanel."
+  spec.email         = ["h.yoshihara@everyleaf.com", "k@uu59.org"]
+  spec.licenses      = ["Apache2"]
+  spec.homepage      = "https://github.com/treasure-data/embulk-input-mixpanel"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'httpclient'
+  spec.add_dependency 'tzinfo'
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'embulk', ['>= 0.6.16', '< 1.0']
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'test-unit'
+  spec.add_development_dependency 'test-unit-rr'
+  spec.add_development_dependency 'codeclimate-test-reporter'
+end

data/gemfiles/embulk-0.6.16

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.16"

data/gemfiles/embulk-0.6.17

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.17"

data/gemfiles/embulk-latest

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'

data/lib/embulk/input/mixpanel.rb

@@ -0,0 +1,165 @@
+require "tzinfo"
+require "embulk/input/mixpanel_api/client"
+
+module Embulk
+  module Input
+
+    class Mixpanel < InputPlugin
+      Plugin.register_input("mixpanel", self)
+
+      GUESS_RECORDS_COUNT = 10
+
+      # NOTE: It takes long time to fetch data between from_date to
+      # to_date by one API request. So this plugin fetches data
+      # between each 7 (SLICE_DAYS_COUNT) days.
+      SLICE_DAYS_COUNT = 7
+
+      def self.transaction(config, &control)
+        task = {}
+
+        task[:params] = export_params(config)
+
+        from_date = config.param(:from_date, :string)
+        to_date = config.param(:to_date, :string)
+        dates = Date.parse(from_date)..Date.parse(to_date)
+        task[:dates] = dates.map {|date| date.to_s}
+
+        task[:api_key] = config.param(:api_key, :string)
+        task[:api_secret] = config.param(:api_secret, :string)
+        task[:timezone] = config.param(:timezone, :string)
+
+        begin
+          # raises exception if timezone is invalid string
+          TZInfo::Timezone.get(task[:timezone])
+        rescue => e
+          Embulk.logger.error "'#{task[:timezone]}' is invalid timezone"
+          raise e
+        end
+
+        columns = []
+        task[:schema] = config.param(:columns, :array)
+        task[:schema].each do |column|
+          name = column["name"]
+          type = column["type"].to_sym
+
+          columns << Column.new(nil, name, type, column["format"])
+        end
+
+        resume(task, columns, 1, &control)
+      end
+
+      def self.resume(task, columns, count, &control)
+        commit_reports = yield(task, columns, count)
+
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      def self.guess(config)
+        client = MixpanelApi::Client.new(config.param(:api_key, :string), config.param(:api_secret, :string))
+
+        from_date = config.param(:from_date, :string)
+        # NOTE: It should have 7 days beteen from_date and to_date
+        to_date = (Date.parse(from_date) + SLICE_DAYS_COUNT - 1).to_s
+
+        params = export_params(config)
+        params = params.merge(
+          from_date: from_date,
+          to_date: to_date,
+        )
+
+        records = client.export(params)
+        sample_records = records.first(GUESS_RECORDS_COUNT)
+        properties = Guess::SchemaGuess.from_hash_records(sample_records.map{|r| r["properties"]})
+        columns = properties.map do |col|
+          result = {
+            name: col.name,
+            type: col.type,
+          }
+          result[:format] = col.format if col.format
+          result
+        end
+        columns.unshift(name: "event", type: :string)
+        return {"columns" => columns}
+      end
+
+      def init
+        @api_key = task[:api_key]
+        @api_secret = task[:api_secret]
+        @params = task[:params]
+        @timezone = task[:timezone]
+        @schema = task[:schema]
+        @dates = task[:dates]
+      end
+
+      def run
+        client = MixpanelApi::Client.new(@api_key, @api_secret)
+        @dates.each_slice(SLICE_DAYS_COUNT) do |dates|
+          from_date = dates.first
+          to_date = dates.last
+          Embulk.logger.info "Fetching data from #{from_date} to #{to_date} ..."
+
+          params = @params.merge(
+            "from_date" => from_date,
+            "to_date" => to_date
+          )
+
+          records = client.export(params)
+
+          records.each do |record|
+            values = @schema.map do |column|
+              case column["name"]
+              when "event"
+                record["event"]
+              when "time"
+                time = record["properties"]["time"]
+                adjust_timezone(time)
+              else
+                record["properties"][column["name"]]
+              end
+            end
+            page_builder.add(values)
+          end
+
+          break if preview?
+        end
+
+        page_builder.finish
+
+        commit_report = {}
+        return commit_report
+      end
+
+      private
+
+      def adjust_timezone(epoch)
+        # Adjust timezone offset to get UTC time
+        # c.f. https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel#export
+        tz = TZInfo::Timezone.get(@timezone)
+        offset = tz.period_for_local(epoch, true).offset.utc_offset
+        epoch - offset
+      end
+
+      def preview?
+        begin
+          org.embulk.spi.Exec.isPreview()
+        rescue java.lang.NullPointerException => e
+          false
+        end
+      end
+
+      def self.export_params(config)
+        event = config.param(:event, :array, default: nil)
+        event = event.nil? ? nil : event.to_json
+
+        {
+          api_key: config.param(:api_key, :string),
+          event: event,
+          where: config.param(:where, :string, default: nil),
+          bucket: config.param(:bucket, :string, default: nil),
+        }
+      end
+    end
+
+  end
+end

data/lib/embulk/input/mixpanel_api/client.rb

@@ -0,0 +1,62 @@
+require "uri"
+require "digest/md5"
+require "json"
+require "httpclient"
+
+module Embulk
+  module Input
+    module MixpanelApi
+      class Client
+        ENDPOINT_EXPORT = "https://data.mixpanel.com/api/2.0/export/".freeze
+        TIMEOUT_SECONDS = 3600
+
+        def initialize(api_key, api_secret)
+          @api_key = api_key
+          @api_secret = api_secret
+        end
+
+        def export(params = {})
+          # https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel
+          params[:expire] ||= Time.now.to_i + TIMEOUT_SECONDS
+          params[:sig] = signature(params)
+          response = httpclient.get(ENDPOINT_EXPORT, params)
+
+          if response.code >= 400
+            Embulk.logger.error response.body
+            return Enumerator.new{ }
+          end
+
+          Enumerator.new do |y|
+            response.body.lines.each do |json|
+              y << JSON.parse(json)
+            end
+          end
+        end
+
+        private
+
+        def signature(params)
+          # https://mixpanel.com/docs/api-documentation/data-export-api#auth-implementation
+          sorted_keys = params.keys.map(&:to_s).sort
+          signature = sorted_keys.inject("") do |sig, key|
+            value = params[key] || params[key.to_sym]
+            next sig unless value
+            sig << "#{key}=#{value}"
+          end
+
+          Digest::MD5.hexdigest(signature + @api_secret)
+        end
+
+        def httpclient
+          @client ||=
+            begin
+              client = HTTPClient.new
+              client.receive_timeout = TIMEOUT_SECONDS
+              client.default_header = {Accept: "application/json; charset=UTF-8"}
+              client
+            end
+        end
+      end
+    end
+  end
+end

data/test/embulk/input/mixpanel_api/test_client.rb

@@ -0,0 +1,130 @@
+require "embulk/input/mixpanel_api/client"
+
+module Embulk
+  module Input
+    module MixpanelApi
+      class ClientTest < Test::Unit::TestCase
+        API_KEY = "api_key".freeze
+        API_SECRET = "api_secret".freeze
+
+        def setup
+          @client = Client.new(API_KEY, API_SECRET)
+        end
+
+        # NOTE: Client#signature is private method but this value
+        # can't be checked via other methods.
+        def test_signature
+          now = Time.parse("2015-07-22 00:00:00")
+          stub(Time).now { now }
+
+          params = {
+            string: "string",
+            array: ["elem1", "elem2"],
+          }
+          expected = "4be4a4f92f57e12b543a2a5f2f5897b6"
+
+          assert_equal(expected, @client.__send__(:signature, params))
+        end
+
+        class ExportTest < self
+          def setup
+            super
+
+            @httpclient = HTTPClient.new
+            stub(@client).httpclient { @httpclient }
+          end
+
+          def test_response_class
+            stub_response(success_response)
+
+            actual = @client.export(params)
+
+            assert_equal(Enumerator, actual.class)
+          end
+
+          def test_http_request
+            mock(@httpclient).get(Client::ENDPOINT_EXPORT, params) do
+              success_response
+            end
+
+            @client.export(params)
+          end
+
+          def test_success
+            stub_response(success_response)
+
+            actual = @client.export(params)
+
+            assert_equal(dummy_responses, actual.to_a)
+          end
+
+          def test_failure
+            stub_response(failure_response)
+
+            stub(Embulk.logger).error(failure_response.body) {}
+            actual = @client.export(params)
+
+            assert_equal([], actual.to_a)
+          end
+
+          def test_failure_logging
+            stub_response(failure_response)
+
+            mock(Embulk.logger).error(failure_response.body) {}
+            @client.export(params)
+          end
+
+          private
+
+          def stub_response(response)
+            stub(@httpclient).get(Client::ENDPOINT_EXPORT, params) do
+              response
+            end
+          end
+
+          def success_response
+            Struct.new(:code, :body).new(200, jsonl_dummy_responses)
+          end
+
+          def failure_response
+            Struct.new(:code, :body).new(400, "{'error': 'invalid'}")
+          end
+
+          def params
+            {
+              api_key: API_KEY,
+              api_secret: API_SECRET,
+              from_date: "2015-01-01",
+              to_date: "2015-03-02",
+            }
+          end
+
+          def dummy_responses
+            [
+              {
+                "event" => "event",
+                "properties" => {
+                  "foo" => "FOO",
+                  "bar" => "2000-01-01 11:11:11",
+                  "int" => 42,
+                }
+              },
+              {
+                "event" => "event2",
+                "properties" => {
+                  "foo" => "fooooooooo",
+                  "bar" => "1988-12-01 12:11:11",
+                  "int" => 1,
+                }
+              },
+            ]
+          end
+
+          def jsonl_dummy_responses
+            dummy_responses.map{|res| JSON.dump(res)}.join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/test/embulk/input/test_mixpanel.rb

@@ -0,0 +1,175 @@
+require "prepare_embulk"
+require "embulk/input/mixpanel"
+require "json"
+
+module Embulk
+  module Input
+    class MixpanelTest < Test::Unit::TestCase
+      API_KEY = "api_key".freeze
+      API_SECRET = "api_secret".freeze
+      FROM_DATE = "2015-02-22".freeze
+      TO_DATE = "2015-03-02".freeze
+
+      DURATIONS = [
+        {from_date: FROM_DATE, to_date: "2015-02-28"}, # It has 7 days between 2015-02-22 and 2015-02-28
+        {from_date: "2015-03-01", to_date: TO_DATE},
+      ]
+
+      def setup
+        setup_client
+        setup_logger
+      end
+
+      def setup_client
+        params = {
+          api_key: API_KEY,
+          event: nil,
+          where: nil,
+          bucket: nil,
+        }
+
+        any_instance_of(MixpanelApi::Client) do |klass|
+          DURATIONS.each do |duration|
+            from_date = duration[:from_date]
+            to_date = duration[:to_date]
+
+            stub(klass).export(params) { records }
+          end
+        end
+      end
+
+      def setup_logger
+        stub(Embulk).logger { ::Logger.new(IO::NULL) }
+      end
+
+      def test_guess
+        expected = {
+          "columns" => [
+            {name: "event", type: :string},
+            {name: "foo", type: :string},
+            {name: "time", type: :long},
+            {name: "int", type: :long},
+          ]
+        }
+
+        actual = Mixpanel.guess(embulk_config)
+        assert_equal(expected, actual)
+      end
+
+      def test_export_params
+        config_params = [
+          :type, "mixpanel",
+          :api_key, API_KEY,
+          :api_secret, API_SECRET,
+          :from_date, FROM_DATE,
+          :to_date, TO_DATE,
+          :event, ["ViewHoge", "ViewFuga"],
+          :where, 'properties["$os"] == "Windows"',
+          :bucket, "987",
+        ]
+
+        config = DataSource[*config_params]
+
+        expected = {
+          api_key: API_KEY,
+          event: "[\"ViewHoge\",\"ViewFuga\"]",
+          where: 'properties["$os"] == "Windows"',
+          bucket: "987",
+        }
+        actual = Mixpanel.export_params(config)
+
+        assert_equal(expected, actual)
+      end
+
+      class RunTest < self
+        def setup
+          super
+
+          @page_builder = Object.new
+          @plugin = Mixpanel.new(task, nil, nil, @page_builder)
+        end
+
+        def test_preview
+          stub(@plugin).preview? { true }
+          mock(@page_builder).add(anything).times(records.length)
+          mock(@page_builder).finish
+
+          @plugin.run
+        end
+
+        def test_run
+          stub(@plugin).preview? { false }
+          mock(@page_builder).add(anything).times(records.length * 2)
+          mock(@page_builder).finish
+
+          @plugin.run
+        end
+
+        def test_timezone
+          stub(@plugin).preview? { false }
+          adjusted = record_epoch - timezone_offset_seconds
+          mock(@page_builder).add(["FOO", adjusted]).times(records.length * 2)
+          mock(@page_builder).finish
+
+          @plugin.run
+        end
+
+        def test_invalid_timezone
+          assert_raise(TZInfo::InvalidTimezoneIdentifier) do
+            Mixpanel.new(task.merge(timezone: "Asia/Tokyooooooooo"), nil, nil, @page_builder).run
+          end
+        end
+
+        private
+
+        def task
+          {
+            api_key: API_KEY,
+            api_secret: API_SECRET,
+            timezone: "Asia/Tokyo",
+            schema: [
+              {"name" => "foo", "type" => "long"},
+              {"name" => "time", "type" => "long"},
+            ],
+            dates: (Date.parse(FROM_DATE)..Date.parse(TO_DATE)).to_a,
+            params: Mixpanel.export_params(embulk_config),
+          }
+        end
+
+        def timezone_offset_seconds
+          60 * 60 * 9 # Asia/Tokyo
+        end
+      end
+
+      private
+
+      def records
+        [
+          {
+            "event" => "event",
+            "properties" => {
+              "foo" => "FOO",
+              "time" => record_epoch,
+              "int" => 42,
+            }
+          },
+        ] * 30
+      end
+
+      def record_epoch
+        1234567890
+      end
+
+      def embulk_config
+        config = {
+          type: "mixpanel",
+          api_key: API_KEY,
+          api_secret: API_SECRET,
+          from_date: FROM_DATE,
+          to_date: TO_DATE,
+        }
+        DataSource[*config.to_a.flatten(1)]
+      end
+    end
+  end
+end

data/test/prepare_embulk.rb

@@ -0,0 +1,10 @@
+require "embulk/command/embulk_run"
+
+classpath_dir = Embulk.home("classpath")
+jars = Dir.entries(classpath_dir).select{|f| f =~ /\.jar$/ }.sort
+jars.each do |jar|
+  require File.join(classpath_dir, jar)
+end
+require "embulk/java/bootstrap"
+
+require "embulk"

data/test/run-test.rb

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+lib_dir = File.join(base_dir, "lib")
+test_dir = File.join(base_dir, "test")
+
+require "test-unit"
+require "test/unit/rr"
+require "codeclimate-test-reporter"
+
+$LOAD_PATH.unshift(lib_dir)
+$LOAD_PATH.unshift(test_dir)
+
+ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
+
+CodeClimate::TestReporter.start
+
+exit Test::Unit::AutoRunner.run(true, test_dir)

metadata

@@ -0,0 +1,199 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-mixpanel
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- yoshihara
+- uu59
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-07-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: httpclient
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: tzinfo
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.16
+    - - <
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: embulk
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.16
+    - - <
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: pry
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit-rr
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: codeclimate-test-reporter
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Loads records from Mixpanel.
+email:
+- h.yoshihara@everyleaf.com
+- k@uu59.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- embulk-input-mixpanel.gemspec
+- gemfiles/embulk-0.6.16
+- gemfiles/embulk-0.6.17
+- gemfiles/embulk-latest
+- lib/embulk/input/mixpanel.rb
+- lib/embulk/input/mixpanel_api/client.rb
+- test/embulk/input/mixpanel_api/test_client.rb
+- test/embulk/input/test_mixpanel.rb
+- test/prepare_embulk.rb
+- test/run-test.rb
+homepage: https://github.com/treasure-data/embulk-input-mixpanel
+licenses:
+- Apache2
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.6
+signing_key:
+specification_version: 4
+summary: Mixpanel input plugin for Embulk
+test_files:
+- test/embulk/input/mixpanel_api/test_client.rb
+- test/embulk/input/test_mixpanel.rb
+- test/prepare_embulk.rb
+- test/run-test.rb