fastly_fluent 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 596ba6b345d184d58dff80164c004effe722f8a0
4
+ data.tar.gz: 8aa51e80eb779e47a06098210b661dc3ebe3423d
5
+ SHA512:
6
+ metadata.gz: caa9f80da2e44ef1067cf750eeebcfa1104319909e6956a9ce06e1104584220e5d4e257da159b7c0f94b87dfd18741226e037aca3c3daf99e44097ebfc5d4982
7
+ data.tar.gz: 711dff47e3a0dd7218ef4f3862e93e43244d718475798c41f9747d99efab9a32db8b4770c3264362e9817858384eeb4ab72083e46693d94025c3d8e40a2baade
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)\
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fastly_fluent"
7
- spec.version = '0.0.2'
7
+ spec.version = '0.0.3'
8
8
  spec.authors = ["Benjamin Bryant"]
9
9
  spec.email = ["benjaminhbryant@gmail.com"]
10
10
  spec.description = %q{fluent plugin for JSON encoded fastly syslogs}
@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
20
20
  spec.add_development_dependency "bundler", "~> 1.3"
21
21
  spec.add_development_dependency "rake"
22
22
  spec.add_runtime_dependency "fluentd"
23
+ spec.add_runtime_dependency "useragent"
23
24
  end
@@ -30,6 +30,7 @@ module Fluent
30
30
  require 'cool.io'
31
31
  require 'fluent/plugin/socket_util'
32
32
  require 'cgi'
33
+ require 'useragent'
33
34
  end
34
35
 
35
36
  config_param :port, :integer, :default => 5140
@@ -97,27 +98,16 @@ module Fluent
97
98
  end
98
99
 
99
100
  ## parse url
100
- if message['url']
101
- uri = URI.parse(message['url'])
102
- record['path'] = uri.path
103
-
104
-
105
- # emit query params
106
- unless uri.query.nil? || uri.query == ""
107
-
108
- CGI::parse(uri.query).each do |k,v|
109
- if record[k].nil?
110
- if v.count > 1
111
- record[k] = v
112
- elsif v.count == 1
113
- record[k] = v.first
114
- else
115
- record[k] = true
116
- end
117
- end
118
- end
119
- end
120
- end
101
+ record = parse_url(message['url']).merge(record) if message['url']
102
+
103
+ record = parse_url(message['referer'], prefix: 'referer_', no_params: true).merge(record) if message['referer']
104
+
105
+ record = parse_user_agent(message['user_agent']).merge(record) if message['user_agent']
106
+
107
+
108
+
109
+
110
+
121
111
 
122
112
 
123
113
  emit(tag, time, record)
@@ -146,5 +136,65 @@ module Fluent
146
136
  rescue => e
147
137
  log.error "fastly failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
148
138
  end
139
+
140
+ def parse_url(url, options = {})
141
+
142
+ out = {}
143
+ uri = URI.parse(url)
144
+
145
+ key_prefix = options[:prefix] || ""
146
+ no_params = options[:no_params] || false
147
+
148
+
149
+ out[key_prefix + 'scheme'] = uri.scheme if uri.scheme
150
+ out[key_prefix + 'host'] = uri.host if uri.host
151
+ out[key_prefix + 'path'] = uri.path if uri.path
152
+ #out[key_prefix + 'fragment'] = uri.fragment if uri.fragment
153
+
154
+
155
+
156
+ # emit query params
157
+ unless no_params || uri.query.nil? || uri.query == ""
158
+
159
+
160
+ CGI::parse(uri.query).each do |k,v|
161
+
162
+ prefix_key = key_prefix + k
163
+
164
+ if out[prefix_key].nil?
165
+
166
+ if v.count > 1
167
+ out[prefix_key] = v
168
+ elsif v.count == 1
169
+ out[prefix_key] = v.first
170
+ else
171
+ out[prefix_key] = true
172
+ end
173
+ end
174
+ end
175
+ end
176
+
177
+ out
178
+ rescue => e
179
+ log.error "url parse error", :error => e.to_s
180
+ out
181
+ end
182
+
183
+ def parse_user_agent(ua)
184
+ out = {}
185
+
186
+ p = UserAgent.parse(ua)
187
+
188
+ out["user_agent_browser"] = p.browser
189
+ out["user_agent_platform"] = p.platform
190
+ out["user_agent_version"] = p.version.to_s
191
+ out["user_agent_type"] = p.mobile? ? "mobile" : (p.bot? ? "bot" : "desktop")
192
+
193
+ out
194
+ rescue => e
195
+ log.error "user agent parse error", :error => e.to_s
196
+ out
197
+ end
198
+
149
199
  end
150
200
  end
@@ -77,7 +77,7 @@ class FastlyInputTest < Test::Unit::TestCase
77
77
  configs.each_pair { |k, v|
78
78
  d = create_driver(v)
79
79
 
80
- test = "<134>2014-07-10T23:18:15Z cache-hk91 td.server.requests[11226]: {\"ip\":\"166.137.213.198\",\"client_id\":\"17a8e5f8f64a9a7c85d1ccab51bcfdf6\",\"status\":200,\"user_agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53\",\"null_val\":\"(null)\",\"url\":\"/my/path?query=param&bob=first&bob=second&sally\"}"
80
+ test = "<134>2014-07-10T23:18:15Z cache-hk91 td.server.requests[11226]: {\"ip\":\"166.137.213.198\",\"client_id\":\"17a8e5f8f64a9a7c85d1ccab51bcfdf6\",\"status\":200,\"user_agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53\",\"null_val\":\"(null)\",\"url\":\"/my/path?query=param&bob=first&bob=second&sally&status=300\",\"referer\":\"https://m.facebook.com/l.php?u=https%3A%2F%2Fgetpebble.com%2F%3Futm_source%3Dfacebook%26utm_medium%3Dcpc%26utm_content%3Dxxx%26utm_campaign%3Dxxx&h=hAQGU-qkW&s=1\"}"
81
81
 
82
82
 
83
83
 
@@ -89,7 +89,7 @@ class FastlyInputTest < Test::Unit::TestCase
89
89
  sleep 1
90
90
  end
91
91
 
92
- tag =d.emits.first[0]
92
+ tag = d.emits.first[0]
93
93
  time = d.emits.first[1]
94
94
  record = d.emits.first[2]
95
95
 
@@ -102,6 +102,20 @@ class FastlyInputTest < Test::Unit::TestCase
102
102
  assert_equal(record['bob'], ["first","second"])
103
103
  assert_equal(record['sally'], true)
104
104
 
105
+
106
+ assert_equal(record['status'], 200) ## ensure reverse merge
107
+
108
+ assert_equal(record['referer_host'], "m.facebook.com")
109
+ assert_equal(record['referer_path'], "/l.php")
110
+ assert_equal(record['referer_s'], nil)
111
+
112
+ assert_equal(record['user_agent_platform'], "iPhone")
113
+ assert_equal(record['user_agent_browser'], "Safari")
114
+ assert_equal(record['user_agent_version'], "7.0.3")
115
+ assert_equal(record['user_agent_type'], "mobile")
116
+
117
+
118
+
105
119
  }
106
120
  end
107
121
 
metadata CHANGED
@@ -1,20 +1,18 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fastly_fluent
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
5
- prerelease:
4
+ version: 0.0.3
6
5
  platform: ruby
7
6
  authors:
8
7
  - Benjamin Bryant
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2014-07-14 00:00:00.000000000 Z
11
+ date: 2014-07-21 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: bundler
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
17
  - - ~>
20
18
  - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
22
20
  type: :development
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
24
  - - ~>
28
25
  - !ruby/object:Gem::Version
@@ -30,33 +27,43 @@ dependencies:
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: rake
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
- - - ! '>='
31
+ - - '>='
36
32
  - !ruby/object:Gem::Version
37
33
  version: '0'
38
34
  type: :development
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
- - - ! '>='
38
+ - - '>='
44
39
  - !ruby/object:Gem::Version
45
40
  version: '0'
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: fluentd
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
- - - ! '>='
45
+ - - '>='
52
46
  - !ruby/object:Gem::Version
53
47
  version: '0'
54
48
  type: :runtime
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
- - - ! '>='
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: useragent
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '>='
60
67
  - !ruby/object:Gem::Version
61
68
  version: '0'
62
69
  description: fluent plugin for JSON encoded fastly syslogs
@@ -78,33 +85,26 @@ files:
78
85
  homepage: ''
79
86
  licenses:
80
87
  - MIT
88
+ metadata: {}
81
89
  post_install_message:
82
90
  rdoc_options: []
83
91
  require_paths:
84
92
  - lib
85
93
  required_ruby_version: !ruby/object:Gem::Requirement
86
- none: false
87
94
  requirements:
88
- - - ! '>='
95
+ - - '>='
89
96
  - !ruby/object:Gem::Version
90
97
  version: '0'
91
- segments:
92
- - 0
93
- hash: 4532696612457073463
94
98
  required_rubygems_version: !ruby/object:Gem::Requirement
95
- none: false
96
99
  requirements:
97
- - - ! '>='
100
+ - - '>='
98
101
  - !ruby/object:Gem::Version
99
102
  version: '0'
100
- segments:
101
- - 0
102
- hash: 4532696612457073463
103
103
  requirements: []
104
104
  rubyforge_project:
105
- rubygems_version: 1.8.25
105
+ rubygems_version: 2.0.6
106
106
  signing_key:
107
- specification_version: 3
107
+ specification_version: 4
108
108
  summary: ''
109
109
  test_files:
110
110
  - test/helper.rb