fastly_fluent 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/fastly_fluent.gemspec +2 -1
- data/lib/fluent/plugin/in_fastly.rb +71 -21
- data/test/plugin/test_fastly_fluent.rb +16 -2
- metadata +25 -25
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 596ba6b345d184d58dff80164c004effe722f8a0
|
4
|
+
data.tar.gz: 8aa51e80eb779e47a06098210b661dc3ebe3423d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: caa9f80da2e44ef1067cf750eeebcfa1104319909e6956a9ce06e1104584220e5d4e257da159b7c0f94b87dfd18741226e037aca3c3daf99e44097ebfc5d4982
|
7
|
+
data.tar.gz: 711dff47e3a0dd7218ef4f3862e93e43244d718475798c41f9747d99efab9a32db8b4770c3264362e9817858384eeb4ab72083e46693d94025c3d8e40a2baade
|
data/fastly_fluent.gemspec
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)\
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fastly_fluent"
|
7
|
-
spec.version = '0.0.
|
7
|
+
spec.version = '0.0.3'
|
8
8
|
spec.authors = ["Benjamin Bryant"]
|
9
9
|
spec.email = ["benjaminhbryant@gmail.com"]
|
10
10
|
spec.description = %q{fluent plugin for JSON encoded fastly syslogs}
|
@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.add_development_dependency "bundler", "~> 1.3"
|
21
21
|
spec.add_development_dependency "rake"
|
22
22
|
spec.add_runtime_dependency "fluentd"
|
23
|
+
spec.add_runtime_dependency "useragent"
|
23
24
|
end
|
@@ -30,6 +30,7 @@ module Fluent
|
|
30
30
|
require 'cool.io'
|
31
31
|
require 'fluent/plugin/socket_util'
|
32
32
|
require 'cgi'
|
33
|
+
require 'useragent'
|
33
34
|
end
|
34
35
|
|
35
36
|
config_param :port, :integer, :default => 5140
|
@@ -97,27 +98,16 @@ module Fluent
|
|
97
98
|
end
|
98
99
|
|
99
100
|
## parse url
|
100
|
-
if message['url']
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
if v.count > 1
|
111
|
-
record[k] = v
|
112
|
-
elsif v.count == 1
|
113
|
-
record[k] = v.first
|
114
|
-
else
|
115
|
-
record[k] = true
|
116
|
-
end
|
117
|
-
end
|
118
|
-
end
|
119
|
-
end
|
120
|
-
end
|
101
|
+
record = parse_url(message['url']).merge(record) if message['url']
|
102
|
+
|
103
|
+
record = parse_url(message['referer'], prefix: 'referer_', no_params: true).merge(record) if message['referer']
|
104
|
+
|
105
|
+
record = parse_user_agent(message['user_agent']).merge(record) if message['user_agent']
|
106
|
+
|
107
|
+
|
108
|
+
|
109
|
+
|
110
|
+
|
121
111
|
|
122
112
|
|
123
113
|
emit(tag, time, record)
|
@@ -146,5 +136,65 @@ module Fluent
|
|
146
136
|
rescue => e
|
147
137
|
log.error "fastly failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
|
148
138
|
end
|
139
|
+
|
140
|
+
def parse_url(url, options = {})
|
141
|
+
|
142
|
+
out = {}
|
143
|
+
uri = URI.parse(url)
|
144
|
+
|
145
|
+
key_prefix = options[:prefix] || ""
|
146
|
+
no_params = options[:no_params] || false
|
147
|
+
|
148
|
+
|
149
|
+
out[key_prefix + 'scheme'] = uri.scheme if uri.scheme
|
150
|
+
out[key_prefix + 'host'] = uri.host if uri.host
|
151
|
+
out[key_prefix + 'path'] = uri.path if uri.path
|
152
|
+
#out[key_prefix + 'fragment'] = uri.fragment if uri.fragment
|
153
|
+
|
154
|
+
|
155
|
+
|
156
|
+
# emit query params
|
157
|
+
unless no_params || uri.query.nil? || uri.query == ""
|
158
|
+
|
159
|
+
|
160
|
+
CGI::parse(uri.query).each do |k,v|
|
161
|
+
|
162
|
+
prefix_key = key_prefix + k
|
163
|
+
|
164
|
+
if out[prefix_key].nil?
|
165
|
+
|
166
|
+
if v.count > 1
|
167
|
+
out[prefix_key] = v
|
168
|
+
elsif v.count == 1
|
169
|
+
out[prefix_key] = v.first
|
170
|
+
else
|
171
|
+
out[prefix_key] = true
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
out
|
178
|
+
rescue => e
|
179
|
+
log.error "url parse error", :error => e.to_s
|
180
|
+
out
|
181
|
+
end
|
182
|
+
|
183
|
+
def parse_user_agent(ua)
|
184
|
+
out = {}
|
185
|
+
|
186
|
+
p = UserAgent.parse(ua)
|
187
|
+
|
188
|
+
out["user_agent_browser"] = p.browser
|
189
|
+
out["user_agent_platform"] = p.platform
|
190
|
+
out["user_agent_version"] = p.version.to_s
|
191
|
+
out["user_agent_type"] = p.mobile? ? "mobile" : (p.bot? ? "bot" : "desktop")
|
192
|
+
|
193
|
+
out
|
194
|
+
rescue => e
|
195
|
+
log.error "user agent parse error", :error => e.to_s
|
196
|
+
out
|
197
|
+
end
|
198
|
+
|
149
199
|
end
|
150
200
|
end
|
@@ -77,7 +77,7 @@ class FastlyInputTest < Test::Unit::TestCase
|
|
77
77
|
configs.each_pair { |k, v|
|
78
78
|
d = create_driver(v)
|
79
79
|
|
80
|
-
test = "<134>2014-07-10T23:18:15Z cache-hk91 td.server.requests[11226]: {\"ip\":\"166.137.213.198\",\"client_id\":\"17a8e5f8f64a9a7c85d1ccab51bcfdf6\",\"status\":200,\"user_agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53\",\"null_val\":\"(null)\",\"url\":\"/my/path?query=param&bob=first&bob=second&sally\"}"
|
80
|
+
test = "<134>2014-07-10T23:18:15Z cache-hk91 td.server.requests[11226]: {\"ip\":\"166.137.213.198\",\"client_id\":\"17a8e5f8f64a9a7c85d1ccab51bcfdf6\",\"status\":200,\"user_agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53\",\"null_val\":\"(null)\",\"url\":\"/my/path?query=param&bob=first&bob=second&sally&status=300\",\"referer\":\"https://m.facebook.com/l.php?u=https%3A%2F%2Fgetpebble.com%2F%3Futm_source%3Dfacebook%26utm_medium%3Dcpc%26utm_content%3Dxxx%26utm_campaign%3Dxxx&h=hAQGU-qkW&s=1\"}"
|
81
81
|
|
82
82
|
|
83
83
|
|
@@ -89,7 +89,7 @@ class FastlyInputTest < Test::Unit::TestCase
|
|
89
89
|
sleep 1
|
90
90
|
end
|
91
91
|
|
92
|
-
tag =d.emits.first[0]
|
92
|
+
tag = d.emits.first[0]
|
93
93
|
time = d.emits.first[1]
|
94
94
|
record = d.emits.first[2]
|
95
95
|
|
@@ -102,6 +102,20 @@ class FastlyInputTest < Test::Unit::TestCase
|
|
102
102
|
assert_equal(record['bob'], ["first","second"])
|
103
103
|
assert_equal(record['sally'], true)
|
104
104
|
|
105
|
+
|
106
|
+
assert_equal(record['status'], 200) ## ensure reverse merge
|
107
|
+
|
108
|
+
assert_equal(record['referer_host'], "m.facebook.com")
|
109
|
+
assert_equal(record['referer_path'], "/l.php")
|
110
|
+
assert_equal(record['referer_s'], nil)
|
111
|
+
|
112
|
+
assert_equal(record['user_agent_platform'], "iPhone")
|
113
|
+
assert_equal(record['user_agent_browser'], "Safari")
|
114
|
+
assert_equal(record['user_agent_version'], "7.0.3")
|
115
|
+
assert_equal(record['user_agent_type'], "mobile")
|
116
|
+
|
117
|
+
|
118
|
+
|
105
119
|
}
|
106
120
|
end
|
107
121
|
|
metadata
CHANGED
@@ -1,20 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fastly_fluent
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.3
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Benjamin Bryant
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2014-07-
|
11
|
+
date: 2014-07-21 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: bundler
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
17
|
- - ~>
|
20
18
|
- !ruby/object:Gem::Version
|
@@ -22,7 +20,6 @@ dependencies:
|
|
22
20
|
type: :development
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
24
|
- - ~>
|
28
25
|
- !ruby/object:Gem::Version
|
@@ -30,33 +27,43 @@ dependencies:
|
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: rake
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - '>='
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '0'
|
38
34
|
type: :development
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - '>='
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '0'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: fluentd
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- -
|
45
|
+
- - '>='
|
52
46
|
- !ruby/object:Gem::Version
|
53
47
|
version: '0'
|
54
48
|
type: :runtime
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: useragent
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
60
67
|
- !ruby/object:Gem::Version
|
61
68
|
version: '0'
|
62
69
|
description: fluent plugin for JSON encoded fastly syslogs
|
@@ -78,33 +85,26 @@ files:
|
|
78
85
|
homepage: ''
|
79
86
|
licenses:
|
80
87
|
- MIT
|
88
|
+
metadata: {}
|
81
89
|
post_install_message:
|
82
90
|
rdoc_options: []
|
83
91
|
require_paths:
|
84
92
|
- lib
|
85
93
|
required_ruby_version: !ruby/object:Gem::Requirement
|
86
|
-
none: false
|
87
94
|
requirements:
|
88
|
-
- -
|
95
|
+
- - '>='
|
89
96
|
- !ruby/object:Gem::Version
|
90
97
|
version: '0'
|
91
|
-
segments:
|
92
|
-
- 0
|
93
|
-
hash: 4532696612457073463
|
94
98
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
95
|
-
none: false
|
96
99
|
requirements:
|
97
|
-
- -
|
100
|
+
- - '>='
|
98
101
|
- !ruby/object:Gem::Version
|
99
102
|
version: '0'
|
100
|
-
segments:
|
101
|
-
- 0
|
102
|
-
hash: 4532696612457073463
|
103
103
|
requirements: []
|
104
104
|
rubyforge_project:
|
105
|
-
rubygems_version:
|
105
|
+
rubygems_version: 2.0.6
|
106
106
|
signing_key:
|
107
|
-
specification_version:
|
107
|
+
specification_version: 4
|
108
108
|
summary: ''
|
109
109
|
test_files:
|
110
110
|
- test/helper.rb
|