fluent-plugin-referer-parser 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/fluent-plugin-referer-parser.gemspec +1 -1
  3. data/lib/fluent/plugin/.rubocop.yml +4 -1
  4. data/lib/fluent/plugin/out_referer_parser.rb +20 -8
  5. data/test/plugin/test_out_referer_parser.rb +38 -11
  6. metadata +1 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 56f8bf0b194d231cd18480ccb8eb565b3f96da29
4
- data.tar.gz: 653fbea2b2c809b8c027070ef1f86be948b9f728
3
+ metadata.gz: 83be55d1792cd02e85a4a9b5bba63d745ea4355c
4
+ data.tar.gz: 9243e20f0358b066528a72fe37d5d643f81e621e
5
5
  SHA512:
6
- metadata.gz: af9bd61a4764e99dbc3ee371650bed74c4d26cbea11affb67bd01d82731dcb1e642367dda4d9c6c872fcf0330b70bd50137d39af6f8533f9f79e3c9a5c047cdb
7
- data.tar.gz: c8f624d327d091b6b55535c044e8de72dca2d349ca176dd92f925a348098ae676ef65254e173b7e74747b15d26572f8fa9c26db06b126866af2daae6aa55ec52
6
+ metadata.gz: 68f5e42275147e0283258a26c9dbbaa5c805c98cd95d4b29325ad4fd36a415ef79304450e9d2f7eb159317f9c45055ffc664d8efc191a2a9e5289df39d294b90
7
+ data.tar.gz: 5f80384e7b38990169ae9534f5d2b871ff705e04fafcc954827532b5ff75c9556a5eb4fb645f0a8c2e8b0df74f3f789da057427d5d744b270e76d59d239702b7
data/fluent-plugin-referer-parser.gemspec CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |gem|
4
4
  gem.name = 'fluent-plugin-referer-parser'
5
- gem.version = '0.0.1'
5
+ gem.version = '0.0.2'
6
6
  gem.authors = ['TAGOMORI Satoshi', 'HARUYAMA Seigo']
7
7
  gem.email = ['tagomoris@gmail.com', 'haruyama@unixuser.org']
8
8
  gem.description = %q{parsing by referer-parser. See: https://github.com/snowplow/referer-parser}
data/lib/fluent/plugin/.rubocop.yml CHANGED
@@ -6,4 +6,7 @@ CyclomaticComplexity:
6
6
 
7
7
  MethodLength:
8
8
  CountComments: false # count full line comments?
9
- Max: 20
9
+ Max: 50
10
+
11
+ GlobalVars:
12
+ Enabled: false
data/lib/fluent/plugin/out_referer_parser.rb CHANGED
@@ -15,11 +15,11 @@ class Fluent::RefererParserOutput < Fluent::Output
15
15
  config_param :out_key_referer, :string, default: 'referer_referer'
16
16
  config_param :out_key_search_term, :string, default: 'referer_search_term'
17
17
 
18
- UNKOWWN_STRING = 'UNKNOWN'
19
- PARSE_ERROR_STRUCT = OpenStruct.new(known: false)
18
+ PARSE_ERROR_STRUCT = OpenStruct.new(known?: false)
20
19
 
21
20
  def initialize
22
21
  super
22
+ require 'cgi'
23
23
  require 'referer-parser'
24
24
  end
25
25
 
@@ -61,17 +61,29 @@ class Fluent::RefererParserOutput < Fluent::Output
61
61
  def emit(tag, es, chain)
62
62
  tag = tag_mangle(tag)
63
63
  es.each do |time, record|
64
- parsed =
64
+ referer =
65
65
  begin
66
66
  RefererParser::Referer.new(record[@key_name])
67
67
  rescue
68
68
  PARSE_ERROR_STRUCT
69
69
  end
70
- record.merge!(
71
- @out_key_known => parsed.known,
72
- @out_key_referer => parsed.referer || UNKOWWN_STRING,
73
- @out_key_search_term => parsed.search_term || UNKOWWN_STRING,
74
- )
70
+ if referer.known?
71
+ search_term = referer.search_term
72
+ parameters = CGI.parse(referer.uri.query)
73
+ input_encoding = parameters['ie'][0] || parameters['ei'][0]
74
+ begin
75
+ search_term = search_term.force_encoding(input_encoding).encode('utf-8') if input_encoding && /utf-?8/i !~ input_encoding
76
+ rescue
77
+ $log.error('invalid referer: ' + referer.uri.to_s)
78
+ end
79
+ record.merge!(
80
+ @out_key_known => true,
81
+ @out_key_referer => referer.referer,
82
+ @out_key_search_term => search_term,
83
+ )
84
+ else
85
+ record.merge!(@out_key_known => false)
86
+ end
75
87
  Fluent::Engine.emit(tag, time, record)
76
88
  end
77
89
  chain.next
data/test/plugin/test_out_referer_parser.rb CHANGED
@@ -61,31 +61,57 @@ out_key_search_term ref_search_term
61
61
  d.emit({ 'value' => 0 }, time)
62
62
  d.emit({ 'value' => 1, 'referer' => 'http://www.google.com/search?q=gateway+oracle+cards+denise+linn&hl=en&client=safari' }, time)
63
63
  d.emit({ 'value' => 2, 'referer' => 'http://www.unixuser.org/' }, time)
64
+ d.emit({ 'value' => 3, 'referer' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_JIS&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' }, time)
65
+ d.emit({ 'value' => 4, 'referer' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_J&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' }, time)
66
+ d.emit({ 'value' => 5, 'referer' => 'http://search.yahoo.co.jp/search?p=%E3%81%BB%E3%81%92&aq=-1&oq=&ei=UTF-8&fr=sfp_as&x=wrt' }, time)
64
67
  end
65
68
 
66
69
  emits = d.emits
67
- assert_equal 3, emits.size
70
+ assert_equal 6, emits.size
68
71
  assert_equal 'merged.message', emits[0][0]
69
72
  assert_equal time, emits[0][1]
70
73
 
71
74
  m = emits[0][2]
72
75
  assert_equal 0, m['value']
73
76
  assert_equal false, m['referer_known']
74
- assert_equal 'UNKNOWN', m['referer_referer']
75
- assert_equal 'UNKNOWN', m['referer_search_term']
76
- assert_equal 4, m.keys.size
77
+ assert_nil m['referer_referer']
78
+ assert_nil m['referer_search_term']
79
+ assert_equal 2, m.keys.size
77
80
 
78
81
  m = emits[1][2]
79
82
  assert_equal 1, m['value']
80
83
  assert_equal true, m['referer_known']
81
84
  assert_equal 'Google', m['referer_referer']
82
85
  assert_equal 'gateway oracle cards denise linn', m['referer_search_term']
86
+ assert_equal 5, m.keys.size
83
87
 
84
88
  m = emits[2][2]
85
89
  assert_equal 2, m['value']
86
90
  assert_equal false, m['referer_known']
87
- assert_equal 'UNKNOWN', m['referer_referer']
88
- assert_equal 'UNKNOWN', m['referer_search_term']
91
+ assert_nil m['referer_referer']
92
+ assert_nil m['referer_search_term']
93
+ assert_equal 3, m.keys.size
94
+
95
+ m = emits[3][2]
96
+ assert_equal 3, m['value']
97
+ assert_equal true, m['referer_known']
98
+ assert_equal 'Google', m['referer_referer']
99
+ assert_equal 'マルチキャスト　学士論文', m['referer_search_term']
100
+ assert_equal 5, m.keys.size
101
+
102
+ # invalid input_encoding
103
+ m = emits[4][2]
104
+ assert_equal 4, m['value']
105
+ assert_equal true, m['referer_known']
106
+ assert_equal 'Google', m['referer_referer']
107
+ assert_equal 5, m.keys.size
108
+
109
+ m = emits[5][2]
110
+ assert_equal 5, m['value']
111
+ assert_equal true, m['referer_known']
112
+ assert_equal 'Yahoo!', m['referer_referer']
113
+ assert_equal 'ほげ', m['referer_search_term']
114
+ assert_equal 5, m.keys.size
89
115
  end
90
116
 
91
117
  def test_emit2
@@ -106,21 +132,22 @@ out_key_search_term ref_search_term
106
132
  m = emits[0][2]
107
133
  assert_equal 0, m['value']
108
134
  assert_equal false, m['ref_known']
109
- assert_equal 'UNKNOWN', m['ref_referer']
110
- assert_equal 'UNKNOWN', m['ref_search_term']
111
- assert_equal 4, m.keys.size
135
+ assert_nil m['referer_referer']
136
+ assert_nil m['referer_search_term']
137
+ assert_equal 2, m.keys.size
112
138
 
113
139
  m = emits[1][2]
114
140
  assert_equal 1, m['value']
115
141
  assert_equal true, m['ref_known']
116
142
  assert_equal 'Google', m['ref_referer']
117
143
  assert_equal 'gateway oracle cards denise linn', m['ref_search_term']
144
+ assert_equal 5, m.keys.size
118
145
 
119
146
  m = emits[2][2]
120
147
  assert_equal 2, m['value']
121
148
  assert_equal false, m['ref_known']
122
- assert_equal 'UNKNOWN', m['ref_referer']
123
- assert_equal 'UNKNOWN', m['ref_search_term']
149
+ assert_nil m['referer_referer']
150
+ assert_nil m['referer_search_term']
124
151
 
125
152
  m = emits[3][2]
126
153
  assert_equal 3, m['value']
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-referer-parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - TAGOMORI Satoshi