fluent-plugin-viki 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4e25ddf170d217cf078ea8075da0b4045af950dd
+  data.tar.gz: 5466ab22f9020a7c2ba5cdef075327764b547d8b
+SHA512:
+  metadata.gz: 96c81345d50a5c81764f5a5abe1720fad5860e990567972024d80d7ff262bdf55d5a9d2136d87431113821264fa23132626f338e32f8505d2a62c0f8994c52be
+  data.tar.gz: b8c47a3cd5c4a1ef9e073ef9e1951b1dd4792d543f49848b94f9bd9e0c977803b32d06627d1fddf2c95658e1cb60ea05213a3b0f2545dc2ae97780b4550cd7a1

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem
+.byebug_history

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.0.0
+before_install: gem install bundler -v 1.11.2

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-viki.gemspec
+gemspec
+
+gem "maxminddb", "~> 0.1"
+gem "fluentd", "~> 0.12", "< 0.13"

data/README.md

@@ -0,0 +1,36 @@
+# Fluent::Plugin::Viki
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/fluent/plugin/viki`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-viki'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-viki
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/fluent-plugin-viki.
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fluent/viki"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config/fluent.conf

@@ -0,0 +1,116 @@
+# In v1 configuration, type and id are @ prefix parameters.
+# @type and @id are recommended. type and id are still available for backward compatibility
+
+## built-in TCP input
+## $ echo <json> | fluent-cat <tag>
+<source>
+  @type forward
+  @id forward_input
+</source>
+
+## built-in UNIX socket input
+#<source>
+#  @type unix
+#</source>
+
+# HTTP input
+# http://localhost:8888/<tag>?json=<json>
+<source>
+  @type http
+  @id http_input
+
+  port 8888
+</source>
+
+<source>
+  @type viki
+  @id http_viki_input
+  port 8889
+</source>
+
+## File input
+## read apache logs with tag=apache.access
+#<source>
+#  @type tail
+#  format apache
+#  path /var/log/httpd-access.log
+#  tag apache.access
+#</source>
+
+# Listen HTTP for monitoring
+# http://localhost:24220/api/plugins
+# http://localhost:24220/api/plugins?type=TYPE
+# http://localhost:24220/api/plugins?tag=MYTAG
+<source>
+  @type monitor_agent
+  @id monitor_agent_input
+
+  port 24220
+</source>
+
+# Listen DRb for debug
+<source>
+  @type debug_agent
+  @id debug_agent_input
+
+  bind 127.0.0.1
+  port 24230
+</source>
+
+## match tag=apache.access and write to file
+#<match apache.access>
+#  @type file
+#  path /var/log/fluent/access
+#</match>
+
+## match tag=debug.** and dump to console
+<match debug.**>
+  @type stdout
+  @id stdout_output
+</match>
+
+# match tag=system.** and forward to another fluent server
+<match system.**>
+  @type forward
+  @id forward_output
+
+  <server>
+    host 192.168.0.11
+  </server>
+  <secondary>
+    <server>
+      host 192.168.0.12
+    </server>
+  </secondary>
+</match>
+
+## match tag=myapp.** and forward and write to file
+#<match myapp.**>
+#  @type copy
+#  <store>
+#    @type forward
+#    buffer_type file
+#    buffer_path /var/log/fluent/myapp-forward
+#    retry_limit 50
+#    flush_interval 10s
+#    <server>
+#      host 192.168.0.13
+#    </server>
+#  </store>
+#  <store>
+#    @type file
+#    path /var/log/fluent/myapp
+#  </store>
+#</match>
+
+## match fluent's internal events
+#<match fluent.**>
+#  @type null
+#</match>
+
+## match not matched logs and write to file
+#<match **>
+#  @type file
+#  path /var/log/fluent/else
+#  compress gz
+#</match>

data/fluent-plugin-viki.gemspec

@@ -0,0 +1,39 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent/plugin/viki/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-viki"
+  spec.version       = Fluent::Viki::VERSION
+  spec.authors       = ["Zhe Wang"]
+  spec.email         = ["zh3w4ng@gmail.com"]
+
+  spec.summary       = %q{Viki's custom fluentd input plugin.}
+  spec.description   = %q{Viki's custom fluentd input plugin.}
+  spec.homepage      = "https://github.com/viki-org/fluent-plugin-viki"
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.3"
+  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "timecop"
+
+  spec.add_dependency "maxminddb", "~> 0.1"
+  spec.add_dependency "fluentd", "~> 0.12", "< 0.13"
+
+  spec.license = 'Apache-2.0'
+end

data/fluentd.conf

@@ -0,0 +1,53 @@
+<source>
+  type viki
+  port 80
+  respond_with_empty_img true
+  default_tag viki.development.untagged
+</source>
+
+<match **.health_check.**>
+  type null
+</match>
+
+<match **.{development,staging}.**>
+  type copy
+  <store>
+    type stdout
+  </store>
+  <store>
+    type forward
+    <server>
+      host $BROADCASTER_HOST
+      port $BROADCASTER_PORT1
+      weight 100
+    </server>
+    <server>
+      host $BROADCASTER_HOST
+      port $BROADCASTER_PORT2
+      weight 100
+    </server>
+    flush_interval 10s
+    buffer_type file
+    buffer_path /var/log/td-agent/buffer/forward/dev/forward.*.buffer
+  </store>
+</match>
+
+<match **.production.**>
+  type forward
+  <server>
+    host $BROADCASTER_HOST
+    port $BROADCASTER_PORT1
+    weight 100
+  </server>
+  <server>
+    host $BROADCASTER_HOST
+    port $BROADCASTER_PORT2
+    weight 100
+  </server>
+  flush_interval 5s
+  buffer_chunk_limit 4m
+  buffer_queue_limit 512
+
+  buffer_type file
+  buffer_path /var/log/td-agent/buffer/forward/prod/forward.*.buffer
+</match>

data/lib/fluent/plugin/geoip.rb

@@ -0,0 +1,28 @@
+module Fluent
+  module Geoip
+    def country_code_of_ip(ip)
+      iso_code = global_geoip.lookup(ip).country.iso_code
+      iso_code && iso_code.downcase
+    end
+
+    def city_of_ip(ip)
+      city = global_geoip.lookup(ip)
+      record = {}
+      record['dma_code'] = city.location.metro_code
+      record['city_name'] = city.city && city.city.name
+      record['latitude'] = city.location.latitude.to_s
+      record['longitude'] = city.location.longitude.to_s
+      record['postal_code'] = city.postal.code
+      record['region_name'] = city.subdivisions.first && city.subdivisions.first.iso_code
+      record
+    end
+
+    def global_geoip
+      @@global_geoip ||= ::MaxMindDB.new(geoip_db_path)
+    end
+
+    def geoip_db_path
+      '/usr/local/var/GeoIP/GeoIP2-City.mmdb'
+    end
+  end
+end

data/lib/fluent/plugin/viki/version.rb

@@ -0,0 +1,5 @@
+module Fluent
+  module Viki
+    VERSION = "0.1.0"
+  end
+end

data/lib/fluent/plugin/viki.rb

@@ -0,0 +1,184 @@
+require 'rubygems'
+require 'bundler/setup'
+Bundler.require(:default)
+require 'fluent/plugin/in_http'
+require 'fluent/plugin/geoip'
+require "fluent/plugin/viki/version"
+
+module Fluent
+  class HttpVikiInput < HttpInput
+    include Geoip
+    CROSSDOMAIN_XML = <<EOF
+<?xml version="1.0"?>
+<!DOCTYPE cross-domain-policy SYSTEM "http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd">
+<cross-domain-policy>
+  <allow-access-from domain="*" />
+</cross-domain-policy>
+EOF
+
+    Plugin.register_input('viki', self)
+
+    config_param :respond_with_empty_img, :bool, :default => 'true'
+    config_param :default_tag, :default => ''
+
+    def on_request(path_info, params)
+      return_hash = nil
+
+      begin
+        tag = path_info[1..-1].split('/').join('.')
+        return return_200_crossdomain unless tag.downcase.match("crossdomain.xml").nil?
+        tag = @default_tag if tag == '' && @default_tag != ''
+
+        record = params.dup
+
+        time = if record['t_f']
+          t_f = record['t_f']
+          record.delete('t_f')
+          t_f.to_i
+        else
+          Engine.now
+        end
+
+        record['t'] = time.to_s unless record['t']
+
+        set_record_ip(params, record)
+        set_record_domain(record)
+
+        record['uuid'] = record.delete('viki_uuid') if record['viki_uuid']
+        record['content_provider'] = record.delete('type') if record['type']
+        record['device_id'] = record.delete('dev_model') if record['dev_model']
+        record.each { |k, v| record[k] = '' if v == 'null' }
+
+        # generate a unique event id for each event
+        unless record['mid']
+          record['mid'] = params['HTTP_X_REQUEST_ID'] || gen_message_id(time)
+        end
+
+        # rename video_view to minute_view
+        record['event'] = 'minute_view' if record['event'] == 'video_view'
+
+        # rename bottom_subtitle to subtitle_lang, add subtitle_enabled
+        if %w(video_play minute_view).include?(record['event'])
+          update_subtitles(record)
+        end
+
+        # fix xunlei data sending timestamps
+        if record['app_id'] == '100105a'
+          record.delete_if {|key, _|  !!(key =~ /\A[0-9]+{13}\z/) }
+        end
+
+      rescue
+        return return_400
+      end
+
+      begin
+        Engine.emit(tag, time, record)
+      rescue
+        return return_505
+      end
+
+      return_200(return_hash)
+    end
+
+    def update_subtitles(record)
+      record['subtitle_lang'] = record.delete('bottom_subtitle') if record['bottom_subtitle'] and record['subtitle_lang'].nil?
+
+      if record['subtitle_visible'].nil?
+        record['subtitle_visible'] = record['subtitle_lang'] && record['subtitle_lang'].size > 0
+      end
+
+      # manual subtitle set to Chinese for xunlei and letv
+      if %w(100106a 100105a).include?(record['app_id'])
+        record['subtitle_enabled'] = true
+        record['subtitle_lang'] = 'zh'
+      end
+    end
+
+    def remove_exceptions_from_hash(attr_hash, app_id)
+      app_hash = { '100004a' => 'ios', '100005a' => 'android', '65535a' => 'flash', '100017a' => 'android', '100016a' => 'mweb', '100018a' => 'xbox' }
+      source = app_hash[app_id]
+
+      if source.nil?
+        puts "Warning: Could not identify source with app_id: "+app_id
+        return nil
+      end
+
+      attr_hash.each { |k, v|
+        if v.any? && v.include?(source)
+          attr_hash.delete(k)
+        end
+      }
+      attr_hash.keys
+    end
+
+
+    def set_record_domain(record)
+      site = record['site']
+      record['domain'] = site.gsub(/^https?:\/\//, '').gsub(/([^\/]*)(.*)/, '\1').gsub(/^www\./, '') if site
+    end
+
+    def set_record_ip(params, record)
+      record['ip'] ||= params['HTTP_X_FORWARDED_FOR'] || params['REMOTE_ADDR']
+      ips = unless record['ip'].nil?
+        if record['ip'].kind_of? String
+          record['ip'].gsub(' ', ',').split(',')
+        elsif record['ip'].kind_of? Array
+          record['ip']
+            .map {|e| e.split(",").map(&:strip) }
+            .inject([]) {|accum, e| accum + e}
+        end
+      end
+
+      record['country'] = record.delete('country_code') if record['country_code']
+
+      valid_ip, geo_country = resolve_correct_ip(ips)
+
+      record['ip_raw'], record['ip'] = record['ip'], valid_ip
+      record['country'] = geo_country || record['country']
+
+      record.merge! city_of_ip(valid_ip)
+    end
+
+    def resolve_correct_ip(ips)
+      ips.each do |ip|
+        geo_country = country_code_of_ip(ip)
+
+        return [ip, geo_country] unless geo_country.nil?
+      end unless ips.nil?
+
+      [ips.first, nil]
+    end
+
+    def return_505
+      ["500 Internal Server Error", { 'Content-type' => 'text/plain' }, "500 Internal Server Error\n#{$!}\n"]
+    end
+
+    def return_400
+      ["400 Bad Request", { 'Content-type' => 'text/plain' }, "400 Bad Request\n#{$!}\n"]
+    end
+
+    def return_200 json_hash
+      if json_hash
+        ["200 OK", { 'Content-type' => 'application/json' }, json_hash.to_json]
+      else
+        if @respond_with_empty_img
+          ["200 OK", { 'Content-type' => 'image/gif', 'Access-Control-Allow-Origin' => '*' }, "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;"]
+        else
+          ["200 OK", { 'Content-type' => 'text/plain', 'Access-Control-Allow-Origin' => '*' }, ""]
+        end
+      end
+    end
+
+    def return_200_crossdomain
+      ["200 OK", { 'Content-type' => 'text/xml', 'Access-Control-Allow-Origin' => '*', 'Cache-Control' => 'public; max-age=36000' }, CROSSDOMAIN_XML]
+    end
+
+    # Generate a unique id for the event, length: 10+1+5 = 16
+    # It's relatively sortable
+    def gen_message_id time
+      r = rand(36**5).to_s(36)
+      "#{time.to_s}-#{r}"
+    end
+
+  end
+end

metadata

@@ -0,0 +1,163 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-viki
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Zhe Wang
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-02-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: maxminddb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.12'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.12'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '0.13'
+description: Viki's custom fluentd input plugin.
+email:
+- zh3w4ng@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- config/fluent.conf
+- fluent-plugin-viki.gemspec
+- fluentd.conf
+- lib/fluent/plugin/geoip.rb
+- lib/fluent/plugin/viki.rb
+- lib/fluent/plugin/viki/version.rb
+homepage: https://github.com/viki-org/fluent-plugin-viki
+licenses:
+- Apache-2.0
+metadata:
+  allowed_push_host: 'TODO: Set to ''http://mygemserver.com'''
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Viki's custom fluentd input plugin.
+test_files: []