fastly_fluent 0.0.2 → 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 596ba6b345d184d58dff80164c004effe722f8a0
+  data.tar.gz: 8aa51e80eb779e47a06098210b661dc3ebe3423d
+SHA512:
+  metadata.gz: caa9f80da2e44ef1067cf750eeebcfa1104319909e6956a9ce06e1104584220e5d4e257da159b7c0f94b87dfd18741226e037aca3c3daf99e44097ebfc5d4982
+  data.tar.gz: 711dff47e3a0dd7218ef4f3862e93e43244d718475798c41f9747d99efab9a32db8b4770c3264362e9817858384eeb4ab72083e46693d94025c3d8e40a2baade

data/fastly_fluent.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)\
 
 Gem::Specification.new do |spec|
   spec.name          = "fastly_fluent"
-  spec.version       = '0.0.2'
+  spec.version       = '0.0.3'
   spec.authors       = ["Benjamin Bryant"]
   spec.email         = ["benjaminhbryant@gmail.com"]
   spec.description   = %q{fluent plugin for JSON encoded fastly syslogs}
@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
   spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "useragent"
 end

data/lib/fluent/plugin/in_fastly.rb

@@ -30,6 +30,7 @@ module Fluent
       require 'cool.io'
       require 'fluent/plugin/socket_util'
       require 'cgi'
+      require 'useragent'
     end
 
     config_param :port, :integer, :default => 5140
@@ -97,27 +98,16 @@ module Fluent
         end
 
         ## parse url
-        if message['url']
-          uri = URI.parse(message['url'])
-          record['path'] = uri.path
-
-
-          # emit query params
-          unless uri.query.nil? || uri.query == ""
-
-            CGI::parse(uri.query).each do |k,v|
-              if record[k].nil?
-                if v.count > 1
-                  record[k] = v
-                elsif v.count == 1
-                  record[k] = v.first
-                else
-                  record[k] = true
-                end
-              end
-            end
-          end
-        end
+        record = parse_url(message['url']).merge(record) if message['url']
+
+        record = parse_url(message['referer'], prefix: 'referer_', no_params: true).merge(record) if message['referer']
+
+        record = parse_user_agent(message['user_agent']).merge(record) if message['user_agent']
+
+
+
+
+
 
 
         emit(tag, time, record)
@@ -146,5 +136,65 @@ module Fluent
     rescue => e
       log.error "fastly failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
     end
+
+    def parse_url(url, options = {})
+
+      out = {}
+      uri = URI.parse(url)
+
+      key_prefix = options[:prefix] || ""
+      no_params = options[:no_params] || false
+
+
+      out[key_prefix + 'scheme'] = uri.scheme if uri.scheme
+      out[key_prefix + 'host'] = uri.host if uri.host
+      out[key_prefix + 'path'] = uri.path if uri.path
+      #out[key_prefix + 'fragment'] = uri.fragment if uri.fragment
+
+
+
+      # emit query params
+      unless no_params || uri.query.nil? || uri.query == ""
+
+
+        CGI::parse(uri.query).each do |k,v|
+
+          prefix_key = key_prefix + k
+
+          if out[prefix_key].nil?
+
+            if v.count > 1
+              out[prefix_key] = v
+            elsif v.count == 1
+              out[prefix_key] = v.first
+            else
+              out[prefix_key] = true
+            end
+          end
+        end
+      end
+
+      out
+    rescue => e
+      log.error "url parse error", :error => e.to_s
+      out
+    end
+
+    def parse_user_agent(ua)
+      out = {}
+
+      p = UserAgent.parse(ua)
+
+      out["user_agent_browser"] = p.browser
+      out["user_agent_platform"] = p.platform
+      out["user_agent_version"] = p.version.to_s
+      out["user_agent_type"] = p.mobile? ? "mobile" : (p.bot? ? "bot" : "desktop")
+
+      out
+    rescue => e
+      log.error "user agent parse error", :error => e.to_s
+      out
+    end
+
   end
 end

data/test/plugin/test_fastly_fluent.rb

@@ -77,7 +77,7 @@ class FastlyInputTest < Test::Unit::TestCase
     configs.each_pair { |k, v|
       d = create_driver(v)
 
-      test = "<134>2014-07-10T23:18:15Z cache-hk91 td.server.requests[11226]: {\"ip\":\"166.137.213.198\",\"client_id\":\"17a8e5f8f64a9a7c85d1ccab51bcfdf6\",\"status\":200,\"user_agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53\",\"null_val\":\"(null)\",\"url\":\"/my/path?query=param&bob=first&bob=second&sally\"}"
+      test = "<134>2014-07-10T23:18:15Z cache-hk91 td.server.requests[11226]: {\"ip\":\"166.137.213.198\",\"client_id\":\"17a8e5f8f64a9a7c85d1ccab51bcfdf6\",\"status\":200,\"user_agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53\",\"null_val\":\"(null)\",\"url\":\"/my/path?query=param&bob=first&bob=second&sally&status=300\",\"referer\":\"https://m.facebook.com/l.php?u=https%3A%2F%2Fgetpebble.com%2F%3Futm_source%3Dfacebook%26utm_medium%3Dcpc%26utm_content%3Dxxx%26utm_campaign%3Dxxx&h=hAQGU-qkW&s=1\"}"
 
 
 
@@ -89,7 +89,7 @@ class FastlyInputTest < Test::Unit::TestCase
         sleep 1
       end
 
-      tag =d.emits.first[0]
+      tag = d.emits.first[0]
       time = d.emits.first[1]
       record = d.emits.first[2]
 
@@ -102,6 +102,20 @@ class FastlyInputTest < Test::Unit::TestCase
       assert_equal(record['bob'], ["first","second"])
       assert_equal(record['sally'], true)
 
+
+      assert_equal(record['status'], 200) ## ensure reverse merge
+
+      assert_equal(record['referer_host'], "m.facebook.com")
+      assert_equal(record['referer_path'], "/l.php")
+      assert_equal(record['referer_s'], nil)
+
+      assert_equal(record['user_agent_platform'], "iPhone")
+      assert_equal(record['user_agent_browser'], "Safari")
+      assert_equal(record['user_agent_version'], "7.0.3")
+      assert_equal(record['user_agent_type'], "mobile")
+
+
+
     }
   end
 

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: fastly_fluent
 version: !ruby/object:Gem::Version
-  version: 0.0.2
-  prerelease: 
+  version: 0.0.3
 platform: ruby
 authors:
 - Benjamin Bryant
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-14 00:00:00.000000000 Z
+date: 2014-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -30,33 +27,43 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: useragent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: fluent plugin for JSON encoded fastly syslogs
@@ -78,33 +85,26 @@ files:
 homepage: ''
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 4532696612457073463
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 4532696612457073463
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.0.6
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: ''
 test_files:
 - test/helper.rb