fastly_fluent 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/fastly_fluent.gemspec +2 -1
- data/lib/fluent/plugin/in_fastly.rb +71 -21
- data/test/plugin/test_fastly_fluent.rb +16 -2
- metadata +25 -25
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 596ba6b345d184d58dff80164c004effe722f8a0
|
4
|
+
data.tar.gz: 8aa51e80eb779e47a06098210b661dc3ebe3423d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: caa9f80da2e44ef1067cf750eeebcfa1104319909e6956a9ce06e1104584220e5d4e257da159b7c0f94b87dfd18741226e037aca3c3daf99e44097ebfc5d4982
|
7
|
+
data.tar.gz: 711dff47e3a0dd7218ef4f3862e93e43244d718475798c41f9747d99efab9a32db8b4770c3264362e9817858384eeb4ab72083e46693d94025c3d8e40a2baade
|
data/fastly_fluent.gemspec
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)\
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fastly_fluent"
|
7
|
-
spec.version = '0.0.
|
7
|
+
spec.version = '0.0.3'
|
8
8
|
spec.authors = ["Benjamin Bryant"]
|
9
9
|
spec.email = ["benjaminhbryant@gmail.com"]
|
10
10
|
spec.description = %q{fluent plugin for JSON encoded fastly syslogs}
|
@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.add_development_dependency "bundler", "~> 1.3"
|
21
21
|
spec.add_development_dependency "rake"
|
22
22
|
spec.add_runtime_dependency "fluentd"
|
23
|
+
spec.add_runtime_dependency "useragent"
|
23
24
|
end
|
@@ -30,6 +30,7 @@ module Fluent
|
|
30
30
|
require 'cool.io'
|
31
31
|
require 'fluent/plugin/socket_util'
|
32
32
|
require 'cgi'
|
33
|
+
require 'useragent'
|
33
34
|
end
|
34
35
|
|
35
36
|
config_param :port, :integer, :default => 5140
|
@@ -97,27 +98,16 @@ module Fluent
|
|
97
98
|
end
|
98
99
|
|
99
100
|
## parse url
|
100
|
-
if message['url']
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
if v.count > 1
|
111
|
-
record[k] = v
|
112
|
-
elsif v.count == 1
|
113
|
-
record[k] = v.first
|
114
|
-
else
|
115
|
-
record[k] = true
|
116
|
-
end
|
117
|
-
end
|
118
|
-
end
|
119
|
-
end
|
120
|
-
end
|
101
|
+
record = parse_url(message['url']).merge(record) if message['url']
|
102
|
+
|
103
|
+
record = parse_url(message['referer'], prefix: 'referer_', no_params: true).merge(record) if message['referer']
|
104
|
+
|
105
|
+
record = parse_user_agent(message['user_agent']).merge(record) if message['user_agent']
|
106
|
+
|
107
|
+
|
108
|
+
|
109
|
+
|
110
|
+
|
121
111
|
|
122
112
|
|
123
113
|
emit(tag, time, record)
|
@@ -146,5 +136,65 @@ module Fluent
|
|
146
136
|
rescue => e
|
147
137
|
log.error "fastly failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
|
148
138
|
end
|
139
|
+
|
140
|
+
def parse_url(url, options = {})
|
141
|
+
|
142
|
+
out = {}
|
143
|
+
uri = URI.parse(url)
|
144
|
+
|
145
|
+
key_prefix = options[:prefix] || ""
|
146
|
+
no_params = options[:no_params] || false
|
147
|
+
|
148
|
+
|
149
|
+
out[key_prefix + 'scheme'] = uri.scheme if uri.scheme
|
150
|
+
out[key_prefix + 'host'] = uri.host if uri.host
|
151
|
+
out[key_prefix + 'path'] = uri.path if uri.path
|
152
|
+
#out[key_prefix + 'fragment'] = uri.fragment if uri.fragment
|
153
|
+
|
154
|
+
|
155
|
+
|
156
|
+
# emit query params
|
157
|
+
unless no_params || uri.query.nil? || uri.query == ""
|
158
|
+
|
159
|
+
|
160
|
+
CGI::parse(uri.query).each do |k,v|
|
161
|
+
|
162
|
+
prefix_key = key_prefix + k
|
163
|
+
|
164
|
+
if out[prefix_key].nil?
|
165
|
+
|
166
|
+
if v.count > 1
|
167
|
+
out[prefix_key] = v
|
168
|
+
elsif v.count == 1
|
169
|
+
out[prefix_key] = v.first
|
170
|
+
else
|
171
|
+
out[prefix_key] = true
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
out
|
178
|
+
rescue => e
|
179
|
+
log.error "url parse error", :error => e.to_s
|
180
|
+
out
|
181
|
+
end
|
182
|
+
|
183
|
+
def parse_user_agent(ua)
|
184
|
+
out = {}
|
185
|
+
|
186
|
+
p = UserAgent.parse(ua)
|
187
|
+
|
188
|
+
out["user_agent_browser"] = p.browser
|
189
|
+
out["user_agent_platform"] = p.platform
|
190
|
+
out["user_agent_version"] = p.version.to_s
|
191
|
+
out["user_agent_type"] = p.mobile? ? "mobile" : (p.bot? ? "bot" : "desktop")
|
192
|
+
|
193
|
+
out
|
194
|
+
rescue => e
|
195
|
+
log.error "user agent parse error", :error => e.to_s
|
196
|
+
out
|
197
|
+
end
|
198
|
+
|
149
199
|
end
|
150
200
|
end
|
@@ -77,7 +77,7 @@ class FastlyInputTest < Test::Unit::TestCase
|
|
77
77
|
configs.each_pair { |k, v|
|
78
78
|
d = create_driver(v)
|
79
79
|
|
80
|
-
test = "<134>2014-07-10T23:18:15Z cache-hk91 td.server.requests[11226]: {\"ip\":\"166.137.213.198\",\"client_id\":\"17a8e5f8f64a9a7c85d1ccab51bcfdf6\",\"status\":200,\"user_agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53\",\"null_val\":\"(null)\",\"url\":\"/my/path?query=param&bob=first&bob=second&sally\"}"
|
80
|
+
test = "<134>2014-07-10T23:18:15Z cache-hk91 td.server.requests[11226]: {\"ip\":\"166.137.213.198\",\"client_id\":\"17a8e5f8f64a9a7c85d1ccab51bcfdf6\",\"status\":200,\"user_agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53\",\"null_val\":\"(null)\",\"url\":\"/my/path?query=param&bob=first&bob=second&sally&status=300\",\"referer\":\"https://m.facebook.com/l.php?u=https%3A%2F%2Fgetpebble.com%2F%3Futm_source%3Dfacebook%26utm_medium%3Dcpc%26utm_content%3Dxxx%26utm_campaign%3Dxxx&h=hAQGU-qkW&s=1\"}"
|
81
81
|
|
82
82
|
|
83
83
|
|
@@ -89,7 +89,7 @@ class FastlyInputTest < Test::Unit::TestCase
|
|
89
89
|
sleep 1
|
90
90
|
end
|
91
91
|
|
92
|
-
tag =d.emits.first[0]
|
92
|
+
tag = d.emits.first[0]
|
93
93
|
time = d.emits.first[1]
|
94
94
|
record = d.emits.first[2]
|
95
95
|
|
@@ -102,6 +102,20 @@ class FastlyInputTest < Test::Unit::TestCase
|
|
102
102
|
assert_equal(record['bob'], ["first","second"])
|
103
103
|
assert_equal(record['sally'], true)
|
104
104
|
|
105
|
+
|
106
|
+
assert_equal(record['status'], 200) ## ensure reverse merge
|
107
|
+
|
108
|
+
assert_equal(record['referer_host'], "m.facebook.com")
|
109
|
+
assert_equal(record['referer_path'], "/l.php")
|
110
|
+
assert_equal(record['referer_s'], nil)
|
111
|
+
|
112
|
+
assert_equal(record['user_agent_platform'], "iPhone")
|
113
|
+
assert_equal(record['user_agent_browser'], "Safari")
|
114
|
+
assert_equal(record['user_agent_version'], "7.0.3")
|
115
|
+
assert_equal(record['user_agent_type'], "mobile")
|
116
|
+
|
117
|
+
|
118
|
+
|
105
119
|
}
|
106
120
|
end
|
107
121
|
|
metadata
CHANGED
@@ -1,20 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fastly_fluent
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.3
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Benjamin Bryant
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2014-07-
|
11
|
+
date: 2014-07-21 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: bundler
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
17
|
- - ~>
|
20
18
|
- !ruby/object:Gem::Version
|
@@ -22,7 +20,6 @@ dependencies:
|
|
22
20
|
type: :development
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
24
|
- - ~>
|
28
25
|
- !ruby/object:Gem::Version
|
@@ -30,33 +27,43 @@ dependencies:
|
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: rake
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - '>='
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '0'
|
38
34
|
type: :development
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - '>='
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '0'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: fluentd
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- -
|
45
|
+
- - '>='
|
52
46
|
- !ruby/object:Gem::Version
|
53
47
|
version: '0'
|
54
48
|
type: :runtime
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: useragent
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
60
67
|
- !ruby/object:Gem::Version
|
61
68
|
version: '0'
|
62
69
|
description: fluent plugin for JSON encoded fastly syslogs
|
@@ -78,33 +85,26 @@ files:
|
|
78
85
|
homepage: ''
|
79
86
|
licenses:
|
80
87
|
- MIT
|
88
|
+
metadata: {}
|
81
89
|
post_install_message:
|
82
90
|
rdoc_options: []
|
83
91
|
require_paths:
|
84
92
|
- lib
|
85
93
|
required_ruby_version: !ruby/object:Gem::Requirement
|
86
|
-
none: false
|
87
94
|
requirements:
|
88
|
-
- -
|
95
|
+
- - '>='
|
89
96
|
- !ruby/object:Gem::Version
|
90
97
|
version: '0'
|
91
|
-
segments:
|
92
|
-
- 0
|
93
|
-
hash: 4532696612457073463
|
94
98
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
95
|
-
none: false
|
96
99
|
requirements:
|
97
|
-
- -
|
100
|
+
- - '>='
|
98
101
|
- !ruby/object:Gem::Version
|
99
102
|
version: '0'
|
100
|
-
segments:
|
101
|
-
- 0
|
102
|
-
hash: 4532696612457073463
|
103
103
|
requirements: []
|
104
104
|
rubyforge_project:
|
105
|
-
rubygems_version:
|
105
|
+
rubygems_version: 2.0.6
|
106
106
|
signing_key:
|
107
|
-
specification_version:
|
107
|
+
specification_version: 4
|
108
108
|
summary: ''
|
109
109
|
test_files:
|
110
110
|
- test/helper.rb
|