sourmix 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b57a705f8b8c54cbb0b5d9cd86921a8cd9fce4e7
+  data.tar.gz: a4d7d2e672f212479b4dc80a4786ab16511d598b
+SHA512:
+  metadata.gz: 42a196ee0b5426b43bd50a903009679c74365562969196c426bb51c2451942d7fed355bde303c564d078754517e1ffea6e9f1732d89adab502bb3527e289a831
+  data.tar.gz: f67e6ff4da041e5e9ce33e687fd1c068a273f1e1197d0e1a13b9c33a91d1b67c21884848f4df0a390519202de1ad6a8ee909bd1f458c9203a4c1d76d668d404a

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2016 Sean Clemmer
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.

data/Readme.md

@@ -0,0 +1,48 @@
+# SourMix ![Version](https://img.shields.io/gem/v/sourmix.svg?style=flat-square)
+
+Shove [Mixpanel](http://mixpanel.com/) events into [Kafka](http://kafka.apache.org/)
+by way of [Theon](https://github.com/sczizzo/theon).
+
+
+## Installation
+
+You can build a gem from this repository, install Ubuntu packages from the
+[Releases](https://github.com/sczizzo/sourmix/releases) page, or use RubyGems:
+
+    $ gem install sourmix
+
+SourMix shells out to `curl` and `gzip`, so you'll also need those installed.
+
+
+## Usage
+
+Just call for help!
+
+    $ sourmix help
+    Commands:
+      sourmix art                                                                                                      ...
+      sourmix go -d, --date=DATE -k, --mixpanel-api-key=MIXPANEL_API_KEY -s, --mixpanel-api-secret=MIXPANEL_API_SECRET ...
+      sourmix help [COMMAND]                                                                                           ...
+      sourmix version
+
+You're probably most interested in the `go` command:
+
+    Usage:
+      sourmix go -d, --date=DATE -k, --mixpanel-api-key=MIXPANEL_API_KEY -s, --mixpanel-api-secret=MIXPANEL_API_SECRET -t, --theon-uri=THEON_URI
+
+    Options:
+      -d, --date=DATE                                # Date to index (YYYY-mm-dd)
+                                                     # Default: 2016-02-08
+      -k, --mixpanel-api-key=MIXPANEL_API_KEY        # Mixpanel API key
+      -s, --mixpanel-api-secret=MIXPANEL_API_SECRET  # Mixpanel API secret
+      -t, --theon-uri=THEON_URI                      # Theon URI (including auth)
+      -L, [--log=LOG]                                # Log to file instead of STDERR
+      -V, [--debug], [--no-debug]                    # Enable DEBUG-level logging
+
+    Do your thing, SourMix
+
+## Changelog
+
+#### v1.0.0
+
+- Initial implementation

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/bin/sourmix

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 'sourmix'
+SourMix::Main.start ARGV

data/lib/sourmix/main.rb

@@ -0,0 +1,56 @@
+require_relative 'metadata'
+require_relative 'mjolnir'
+require_relative 'perform'
+
+Thread.abort_on_exception = true
+
+
+module SourMix
+  class Main < Mjolnir
+    include Perform
+
+
+    desc 'version', 'Echo the application version'
+    def version
+      puts VERSION
+    end
+
+
+    desc 'art', 'View the application art'
+    def art
+      puts "\n%s\n" % ART
+    end
+
+
+    desc 'go', 'Do your thing, SourMix'
+    option :date, \
+      type: :string,
+      aliases: %w[ -d ],
+      desc: 'Date to index (YYYY-mm-dd)',
+      required: true,
+      default: Time.now.strftime('%Y-%m-%d')
+    option :mixpanel_api_key, \
+      type: :string,
+      aliases: %w[ -k ],
+      desc: 'Mixpanel API key',
+      required: true,
+      default: ENV['SOURMIX_MIXPANEL_API_KEY']
+    option :mixpanel_api_secret, \
+      type: :string,
+      aliases: %w[ -s ],
+      desc: 'Mixpanel API secret',
+      required: true,
+      default: ENV['SOURMIX_MIXPANEL_API_SECRET']
+    option :theon_uri, \
+      type: :string,
+      aliases: %w[ -t ],
+      desc: 'Theon URI (including auth)',
+      required: true,
+      default: ENV['SOURMIX_THEON_URI']
+    include_common_options
+    def go
+      perform
+    end
+
+  end
+end

data/lib/sourmix/metadata.rb

@@ -0,0 +1,42 @@
+module SourMix
+
+  # We use a VERSION file to tie into our build pipeline
+  VERSION  = File.read(File.join(File.dirname(__FILE__), '..', '..', 'VERSION')).strip
+
+  # We don't really do all that much, be humble
+  SUMMARY  = 'Shove Mixpanel events into Kafka by way of Theon'
+
+  # Like the MIT license, but even simpler
+  LICENSE  = 'ISC'
+
+  # Where you should look first
+  HOMEPAGE = 'https://github.com/sczizzo/sourmix'
+
+  # Your benevolent dictator for life
+  AUTHOR   = 'Sean Clemmer'
+
+  # Turn here to strangle your dictator
+  EMAIL    = 'sczizzo@gmail.com'
+
+  # Bundled extensions
+  TRAVELING_RUBY_BUCKET = 'http://d6r77u77i8pq3.cloudfront.net'
+  TRAVELING_RUBY_VERSION = '20150715-2.2.2'
+  JSON_VERSION = '1.8.2'
+
+  # Every project deserves its own ASCII art
+  ART      = <<-'EOART' % VERSION
+
+              _            _      _                  _          _   _          _     _      _
+             / /\         /\ \   /\_\               /\ \       /\_\/\_\ _     /\ \ /_/\    /\ \
+            / /  \       /  \ \ / / /         _    /  \ \     / / / / //\_\   \ \ \\ \ \   \ \_\
+           / / /\ \__   / /\ \ \\ \ \__      /\_\ / /\ \ \   /\ \/ \ \/ / /   /\ \_\\ \ \__/ / /
+          / / /\ \___\ / / /\ \ \\ \___\    / / // / /\ \_\ /  \____\__/ /   / /\/_/ \ \__ \/_/
+          \ \ \ \/___// / /  \ \_\\__  /   / / // / /_/ / // /\/________/   / / /     \/_/\__/\
+           \ \ \     / / /   / / // / /   / / // / /__\/ // / /\/_// / /   / / /       _/\/__\ \
+       _    \ \ \   / / /   / / // / /   / / // / /_____// / /    / / /   / / /       / _/_/\ \ \
+      /_/\__/ / /  / / /___/ / // / /___/ / // / /\ \ \ / / /    / / /___/ / /__     / / /   \ \ \
+      \ \/___/ /  / / /____\/ // / /____\/ // / /  \ \ \\/_/    / / //\__\/_/___\   / / /    /_/ /
+       \_____\/   \/_________/ \/_________/ \/_/    \_\/        \/_/ \/_________/   \/_/     \_\/  v%s
+
+  EOART
+end

data/lib/sourmix/mixpanel_client.rb

@@ -0,0 +1,74 @@
+require 'digest/md5'
+require 'net/http'
+require 'json'
+require 'cgi'
+
+
+module Mixpanel
+  class Client
+    attr_reader :api_key, :api_secret, :base_url, :data_url, :expire_at
+
+    def initialize options
+      @api_key    = options.fetch :api_key
+      @api_secret = options.fetch :api_secret
+      @base_url   = options.fetch :base_url, 'https://mixpanel.com/api/2.0'
+      @data_url   = options.fetch :data_url, 'https://data.mixpanel.com/api/2.0'
+      @expire_at  = options.fetch :expire_at, 90 # seconds
+    end
+
+    def request endpoint, params
+      resp = Net::HTTP.get request_uri(endpoint, params)
+      resp.lines.map { |l| JSON.parse l }
+    end
+
+    def request_uri endpoint, params
+      URI request_url(endpoint, params)
+    end
+
+
+  private
+
+    def request_url endpoint, params
+      api = request_api endpoint
+      url = File.join api, endpoint.to_s
+      sps = sign_params endpoint, params
+      qs  = encode_params sps
+      "#{url}?#{qs}"
+    end
+
+    def request_api endpoint
+      return data_url if endpoint == 'export'
+      return base_url
+    end
+
+    def request_format endpoint
+      return :raw if endpoint == 'export'
+      return :json
+    end
+
+    def encode_params params
+      params.map { |k,v| "#{k}=#{CGI.escape(v.to_s)}" }.sort.join('&')
+    end
+
+    def sign_params endpoint, params
+      nps = normalize_params endpoint, params
+      jps = nps.map { |k,v| "#{k}=#{v}" }.sort.join('')
+      req = jps + api_secret
+      nps[:sig] = Digest::MD5.hexdigest req
+      nps
+    end
+
+    def normalize_params endpoint, params
+      params.merge \
+        api_key: api_key,
+        expire: request_expire(params),
+        format: request_format(endpoint)
+    end
+
+    def request_expire params
+      return params[:expire] if params[:expire]
+      Time.now.to_i + expire_at
+    end
+
+  end
+end

data/lib/sourmix/mjolnir.rb

@@ -0,0 +1,49 @@
+require 'slog'
+require 'thor'
+
+
+# Thor's hammer! Like Thor with better logging
+class Mjolnir < Thor
+
+  # Common options for Thor commands
+  COMMON_OPTIONS = {
+    log: {
+      type: :string,
+      aliases: %w[ -L ],
+      desc: 'Log to file instead of STDERR',
+      default: ENV['SOURMIX_LOG'] || nil
+    },
+    debug: {
+      type: :boolean,
+      aliases: %w[ -V ],
+      desc: 'Enable DEBUG-level logging',
+      default: ENV['SOURMIX_DEBUG'] || false
+    }
+  }
+
+  # Decorate Thor commands with the options above
+  def self.include_common_options
+    COMMON_OPTIONS.each do |name, spec|
+      option name, spec
+    end
+  end
+
+
+  no_commands do
+
+    # Construct a Logger given the command-line options
+    def log
+      return @logger if defined? @logger
+      level = :info
+      level = :debug if options.debug?
+      device = options.log || $stderr
+      pretty = device.tty? rescue false
+      @logger = Slog.new \
+        out: device,
+        level: level,
+        colorize: pretty,
+        prettify: pretty
+    end
+
+  end
+end

data/lib/sourmix/perform.rb

@@ -0,0 +1,181 @@
+require 'shellwords'
+require 'digest/md5'
+require 'tempfile'
+require 'thread'
+require 'logger'
+require 'json'
+require 'date'
+require 'time'
+
+require_relative 'mixpanel_client'
+
+
+
+module SourMix
+  module Perform
+    PAGE_SIZE = 50_000
+
+    def perform
+      mixpanel = Mixpanel::Client.new \
+        api_key: options.mixpanel_api_key,
+        api_secret: options.mixpanel_api_secret
+      dataset = download_dataset mixpanel
+      results = process_dataset dataset
+      gzip_results results
+      report_results results
+    end
+
+
+
+  private
+
+    def download_dataset mixpanel
+      log.debug event: :download_dataset
+
+      dataset = Tempfile.new 'dataset'
+
+      request_uri = mixpanel.request_uri 'export', \
+        from_date: options.date, to_date: options.date
+
+      download_dataset = 'curl -Lf -o %{out} %{url}' % {
+        url: Shellwords.escape(request_uri),
+        out: Shellwords.escape(dataset.path)
+      }
+
+      _, elapsed_s = timed { sh download_dataset }
+
+      log.info event: :download_dataset, exited: $?.exitstatus, elapsed_s: elapsed_s
+
+      dataset
+    end
+
+
+    def process_dataset dataset
+      log.debug event: :process_dataset
+
+      results = []
+      nok, nerr = 0, 0
+
+      _, elapsed_s = timed do
+        dataset.each_line do |e|
+          if nok % PAGE_SIZE == 0
+            log.debug event: :process_dataset_page, okay: nok
+            results << Tempfile.new('results')
+          end
+
+          begin
+            event = JSON.generate(normalize_event(JSON.parse(e)))
+            if v2_api?
+              results.last.puts ':%s' % event
+            else
+              results.last.puts event
+            end
+            nok += 1
+          rescue => e
+            log.error event: :process_dataset_error, error: e.inspect, line: e
+            nerr += 1
+          end
+        end
+      end
+
+      dataset.close
+      dataset.unlink
+      results.map(&:close)
+
+      ops = 1.0 * (nok + nerr) / elapsed_s
+
+      log.info event: :process_dataset, okay: nok, errors: nerr, elapsed_s: elapsed_s, ops: ops, pages: results.size
+
+      results
+    end
+
+
+    def gzip_results results
+      log.debug event: :gzip_results
+      started, total_size = Time.now.to_f, 0
+
+      results.map do |result|
+        Thread.new do
+          gzip_result  = 'gzip -k %s' % Shellwords.escape(result.path)
+          _, elapsed_s = timed { sh gzip_result }
+
+          size = File::Stat.new(result.path + '.gz').size
+          total_size += size
+
+          log.debug event: :gzip_result, exited: $?.exitstatus, elapsed_s: elapsed_s, size: size
+        end
+      end.map(&:join)
+
+      log.info event: :gzip_result, total_elapsed_s: (Time.now.to_f - started), total_size: total_size
+    end
+
+
+    def report_results results
+      log.debug event: :report_results
+
+      theon_uri  = URI options.theon_uri
+      theon_auth = [ theon_uri.user, theon_uri.password ].join(':')
+      theon_url  = '%s://%s:%s%s' % [
+        theon_uri.scheme, theon_uri.host, theon_uri.port, theon_uri.path
+      ]
+
+      auth_opt = theon_auth.nil? ? nil : '-u %s' % Shellwords.escape(theon_auth)
+      started = Time.now.to_f
+
+      results.map do |result|
+        Thread.new do
+          report_result = \
+            'curl %{auth} -Lf %{url} -XPOST %{headers} --data-binary @%{out}.gz' % {
+              auth: auth_opt,
+              url: Shellwords.escape(theon_url),
+              out: Shellwords.escape(result.path),
+              headers: "-H 'Content-Encoding: gzip'"
+            }
+
+          _, elapsed_s = timed { sh report_result }
+          result.unlink
+
+          log.debug event: :report_result, exited: $?.exitstatus, elapsed_s: elapsed_s
+        end
+      end.map(&:join)
+
+      log.info event: :report_results, total_elapsed_s: (Time.now.to_f - started)
+    end
+
+
+    def v2_api? ; options.theon_uri.include? '/v2/' end
+
+
+    def normalize_event e
+      digest = Digest::MD5.hexdigest e.to_s
+      e.merge! e.delete('properties')
+      e = e.inject({}) do |h,(k,v)|
+        k = k.gsub('.','_').gsub(/^\W+/,'')
+        h[k] = v unless v.nil? || v == '' || v == [] || v == {}
+        h
+      end
+      e['@timestamp'] = Time.at(e['time']).utc.iso8601(3)
+      e['@id'] = digest
+      e
+    end
+
+
+    def sh command
+      if options.debug
+        puts command
+        system command
+      else
+        `#{command} 2>&1`
+      end
+    end
+
+
+    def timed &block
+      started = Time.now.to_f
+      rvalue  = yield
+      elapsed = Time.now.to_f - started
+      return rvalue, elapsed
+    end
+
+  end
+end

data/lib/sourmix.rb

@@ -0,0 +1 @@
+require_relative 'sourmix/main'

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: sourmix
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Sean Clemmer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-02-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: slog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+description: Shove Mixpanel events into Kafka by way of Theon.
+email: sczizzo@gmail.com
+executables:
+- sourmix
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- Readme.md
+- VERSION
+- bin/sourmix
+- lib/sourmix.rb
+- lib/sourmix/main.rb
+- lib/sourmix/metadata.rb
+- lib/sourmix/mixpanel_client.rb
+- lib/sourmix/mjolnir.rb
+- lib/sourmix/perform.rb
+homepage: https://github.com/sczizzo/sourmix
+licenses:
+- ISC
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Shove Mixpanel events into Kafka by way of Theon
+test_files: []