fluent-plugin-netflow 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: c83a5f4fb673a2350abaab3f8d33ca43229399e5
4
+ data.tar.gz: 8a24c6097b65b0a7eac8391cc9a54444c4a49a17
5
+ SHA512:
6
+ metadata.gz: ce6510d36beed459299342e64ae6336b004d9c924886336edf57c214ff29405dc96d9764976147d383d192e70bfed7b935e6156f3491678c95fe306628f3fe2b
7
+ data.tar.gz: ce0a4789512e66527d777ae8b6b37958b6d94221874ee91651f8764019095efd308f5c70b44e5f9d980a9cc6c0d643a1e7107eea43997a632765d87978a02e66
data/.travis.yml ADDED
@@ -0,0 +1,19 @@
1
+ language: ruby
2
+
3
+ rvm:
4
+ - 1.9.3
5
+ - 2.0.0
6
+ - 2.1
7
+ - ruby-head
8
+ - rbx
9
+
10
+ branches:
11
+ only:
12
+ - master
13
+
14
+ matrix:
15
+ allow_failures:
16
+ - rvm: ruby-head
17
+ - rvm: rbx
18
+
19
+ script: bundle exec rake test
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec
data/README.md ADDED
@@ -0,0 +1,28 @@
1
+ # Netflow plugin for Fluentd
2
+
3
+ Accept Netflow logs.
4
+
5
+ ## Installation
6
+
7
+ Use RubyGems:
8
+
9
+ gem install fluent-plugin-netflow
10
+
11
+ ## Configuration
12
+
13
+ <source>
14
+ type netflow
15
+ tag netflow.event
16
+
17
+ # optional parameters
18
+ bind 127.0.0.1
19
+ port 5140
20
+
21
+ # optional parser parameters
22
+ cache_ttl 6000
23
+ versions [5, 9]
24
+ </match>
25
+
26
+ ## TODO
27
+
28
+ * Release as rubygem
data/Rakefile ADDED
@@ -0,0 +1,14 @@
1
+
2
+ require 'bundler'
3
+ Bundler::GemHelper.install_tasks
4
+
5
+ require 'rake/testtask'
6
+
7
+ Rake::TestTask.new(:test) do |test|
8
+ test.libs << 'lib' << 'test'
9
+ test.test_files = FileList['test/*.rb']
10
+ test.verbose = true
11
+ end
12
+
13
+ task :default => [:build]
14
+
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.0.1
@@ -0,0 +1,23 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.name = "fluent-plugin-netflow"
6
+ gem.description = "Netflow plugin for Fluentd"
7
+ gem.homepage = "https://github.com/repeatedly/fluent-plugin-netflow"
8
+ gem.summary = gem.description
9
+ gem.version = File.read("VERSION").strip
10
+ gem.authors = ["Masahiro Nakagawa"]
11
+ gem.email = "repeatedly@gmail.com"
12
+ gem.has_rdoc = false
13
+ #gem.platform = Gem::Platform::RUBY
14
+ gem.license = 'Apache License (2.0)'
15
+ gem.files = `git ls-files`.split("\n")
16
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
17
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
+ gem.require_paths = ['lib']
19
+
20
+ gem.add_dependency "fluentd", "~> 0.10.17"
21
+ gem.add_dependency "bindata", "~> 2.1"
22
+ gem.add_development_dependency "rake", ">= 0.9.2"
23
+ end
@@ -0,0 +1,115 @@
1
+ #
2
+ # Fluent
3
+ #
4
+ # Copyright (C) 2014 Masahiro Nakagawa
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ module Fluent
19
+ class NetflowInput < Input
20
+ Plugin.register_input('netflow', self)
21
+
22
+ def initialize
23
+ super
24
+ require 'cool.io'
25
+ require 'fluent/plugin/socket_util'
26
+ require 'fluent/plugin/parser_netflow'
27
+ end
28
+
29
+ config_param :port, :integer, :default => 5140
30
+ config_param :bind, :string, :default => '0.0.0.0'
31
+ config_param :tag, :string
32
+ config_param :protocol_type, :default => :udp do |val|
33
+ case val.downcase
34
+ when 'udp'
35
+ :udp
36
+ else
37
+ raise ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
38
+ end
39
+ end
40
+
41
+ def configure(conf)
42
+ super
43
+
44
+ @parser = TextParser::NetflowParser.new
45
+ @parser.configure(conf)
46
+ end
47
+
48
+ def start
49
+ @loop = Coolio::Loop.new
50
+ @handler = listen(method(:receive_data))
51
+ @loop.attach(@handler)
52
+
53
+ @thread = Thread.new(&method(:run))
54
+ end
55
+
56
+ def shutdown
57
+ @loop.watchers.each { |w| w.detach }
58
+ @loop.stop
59
+ @handler.close
60
+ @thread.join
61
+ end
62
+
63
+ def run
64
+ @loop.run
65
+ rescue
66
+ log.error "unexpected error", :error=>$!.to_s
67
+ log.error_backtrace
68
+ end
69
+
70
+ protected
71
+
72
+ def receive_data(host, data)
73
+ @parser.call(data) { |time, record|
74
+ unless time && record
75
+ log.warn "pattern not match: #{text.inspect}"
76
+ return
77
+ end
78
+
79
+ record['host'] = host
80
+ Engine.emit(tag, time, record)
81
+ }
82
+ rescue
83
+ log.warn data.dump, :error => $!.to_s
84
+ log.debug_backtrace
85
+ end
86
+
87
+ private
88
+
89
+ def listen(callback)
90
+ log.debug "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
91
+ if @protocol_type == :udp
92
+ @usock = SocketUtil.create_udp_socket(@bind)
93
+ @usock.bind(@bind, @port)
94
+ UdpHandler.new(@usock, callback)
95
+ else
96
+ Coolio::TCPServer.new(@bind, @port, TcpHandler, log, callback)
97
+ end
98
+ end
99
+
100
+ class UdpHandler < Coolio::IO
101
+ def initialize(io, callback)
102
+ super(io)
103
+ @io = io
104
+ @callback = callback
105
+ end
106
+
107
+ def on_readable
108
+ msg, addr = @io.recvfrom_nonblock(4096)
109
+ @callback.call(addr[3], msg)
110
+ rescue
111
+ # TODO log?
112
+ end
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,215 @@
1
+ ---
2
+ 1:
3
+ - 4
4
+ - :in_bytes
5
+ 2:
6
+ - 4
7
+ - :in_pkts
8
+ 3:
9
+ - 4
10
+ - :flows
11
+ 4:
12
+ - :uint8
13
+ - :protocol
14
+ 5:
15
+ - :uint8
16
+ - :src_tos
17
+ 6:
18
+ - :uint8
19
+ - :tcp_flags
20
+ 7:
21
+ - :uint16
22
+ - :l4_src_port
23
+ 8:
24
+ - :ip4_addr
25
+ - :ipv4_src_addr
26
+ 9:
27
+ - :uint8
28
+ - :src_mask
29
+ 10:
30
+ - 2
31
+ - :input_snmp
32
+ 11:
33
+ - :uint16
34
+ - :l4_dst_port
35
+ 12:
36
+ - :ip4_addr
37
+ - :ipv4_dst_addr
38
+ 13:
39
+ - :uint8
40
+ - :dst_mask
41
+ 14:
42
+ - 2
43
+ - :output_snmp
44
+ 15:
45
+ - :ip4_addr
46
+ - :ipv4_next_hop
47
+ 16:
48
+ - 2
49
+ - :src_as
50
+ 17:
51
+ - 2
52
+ - :dst_as
53
+ 18:
54
+ - :ip4_addr
55
+ - :bgp_ipv4_next_hop
56
+ 19:
57
+ - 4
58
+ - :mul_dst_pkts
59
+ 20:
60
+ - 4
61
+ - :mul_dst_bytes
62
+ 21:
63
+ - :uint32
64
+ - :last_switched
65
+ 22:
66
+ - :uint32
67
+ - :first_switched
68
+ 23:
69
+ - 4
70
+ - :out_bytes
71
+ 24:
72
+ - 4
73
+ - :out_pkts
74
+ 25:
75
+ - :uint16
76
+ - :min_pkt_length
77
+ 26:
78
+ - :uint16
79
+ - :max_pkt_length
80
+ 27:
81
+ - :ip6_addr
82
+ - :ipv6_src_addr
83
+ 28:
84
+ - :ip6_addr
85
+ - :ipv6_dst_addr
86
+ 29:
87
+ - :uint8
88
+ - :ipv6_src_mask
89
+ 30:
90
+ - :uint8
91
+ - :ipv6_dst_mask
92
+ 31:
93
+ - :uint24
94
+ - :ipv6_flow_label
95
+ 32:
96
+ - :uint16
97
+ - :icmp_type
98
+ 33:
99
+ - :uint8
100
+ - :mul_igmp_type
101
+ 34:
102
+ - :uint32
103
+ - :sampling_interval
104
+ 35:
105
+ - :uint8
106
+ - :sampling_algorithm
107
+ 36:
108
+ - :uint16
109
+ - :flow_active_timeout
110
+ 37:
111
+ - :uint16
112
+ - :flow_inactive_timeout
113
+ 38:
114
+ - :uint8
115
+ - :engine_type
116
+ 39:
117
+ - :uint8
118
+ - :engine_id
119
+ 40:
120
+ - 4
121
+ - :total_bytes_exp
122
+ 41:
123
+ - 4
124
+ - :total_pkts_exp
125
+ 42:
126
+ - 4
127
+ - :total_flows_exp
128
+ 43:
129
+ - :skip
130
+ 44:
131
+ - :ip4_addr
132
+ - :ipv4_src_prefix
133
+ 45:
134
+ - :ip4_addr
135
+ - :ipv4_dst_prefix
136
+ 46:
137
+ - :uint8
138
+ - :mpls_top_label_type
139
+ 47:
140
+ - :uint32
141
+ - :mpls_top_label_ip_addr
142
+ 48:
143
+ - 4
144
+ - :flow_sampler_id
145
+ 49:
146
+ - :uint8
147
+ - :flow_sampler_mode
148
+ 50:
149
+ - :uint32
150
+ - :flow_sampler_random_interval
151
+ 51:
152
+ - :skip
153
+ 52:
154
+ - :uint8
155
+ - :min_ttl
156
+ 53:
157
+ - :uint8
158
+ - :max_ttl
159
+ 54:
160
+ - :uint16
161
+ - :ipv4_ident
162
+ 55:
163
+ - :uint8
164
+ - :dst_tos
165
+ 56:
166
+ - :mac_addr
167
+ - :in_src_max
168
+ 57:
169
+ - :mac_addr
170
+ - :out_dst_max
171
+ 58:
172
+ - :uint16
173
+ - :src_vlan
174
+ 59:
175
+ - :uint16
176
+ - :dst_vlan
177
+ 60:
178
+ - :uint8
179
+ - :ip_protocol_version
180
+ 61:
181
+ - :uint8
182
+ - :direction
183
+ 62:
184
+ - :ip6_addr
185
+ - :ipv6_next_hop
186
+ 63:
187
+ - :ip6_addr
188
+ - :bgp_ipv6_next_hop
189
+ 64:
190
+ - :uint32
191
+ - :ipv6_option_headers
192
+ 64:
193
+ - :skip
194
+ 65:
195
+ - :skip
196
+ 66:
197
+ - :skip
198
+ 67:
199
+ - :skip
200
+ 68:
201
+ - :skip
202
+ 69:
203
+ - :skip
204
+ 80:
205
+ - :mac_addr
206
+ - :in_dst_mac
207
+ 81:
208
+ - :mac_addr
209
+ - :out_src_mac
210
+ 82:
211
+ - :string
212
+ - :if_name
213
+ 83:
214
+ - :string
215
+ - :if_desc
@@ -0,0 +1,448 @@
1
+ require "bindata"
2
+ require "ipaddr"
3
+ require 'yaml'
4
+
5
+ require 'fluent/parser'
6
+
7
+ module Fluent
8
+ class TextParser
9
+ # port from logstash's netflow parser
10
+ class NetflowParser
11
+ include Configurable
12
+
13
+ config_param :cache_ttl, :integer, :default => 4000
14
+ config_param :versions, :default => [5, 9] do |param|
15
+ if param.is_a?(Array)
16
+ param
17
+ else
18
+ param.split(".").map(&:to_i)
19
+ end
20
+ end
21
+ config_param :definitions, :string, :default => nil
22
+
23
+ def configure(conf)
24
+ super
25
+
26
+ @templates = Vash.new()
27
+ # Path to default Netflow v9 field definitions
28
+ filename = File.expand_path('../netflow.yaml', __FILE__)
29
+
30
+ begin
31
+ @fields = YAML.load_file(filename)
32
+ rescue Exception => e
33
+ raise "Bad syntax in definitions file #{filename}"
34
+ end
35
+
36
+ # Allow the user to augment/override/rename the supported Netflow fields
37
+ if @definitions
38
+ raise "definitions file #{@definitions} does not exists" unless File.exist?(@definitions)
39
+ begin
40
+ @fields.merge!(YAML.load_file(@definitions))
41
+ rescue Exception => e
42
+ raise "Bad syntax in definitions file #{@definitions}"
43
+ end
44
+ end
45
+ end
46
+
47
+ def call(payload)
48
+ header = Header.read(payload)
49
+ unless @versions.include?(header.version)
50
+ $log.warn "Ignoring Netflow version v#{header.version}"
51
+ return
52
+ end
53
+
54
+ if header.version == 5
55
+ flowset = Netflow5PDU.read(payload)
56
+ elsif header.version == 9
57
+ flowset = Netflow9PDU.read(payload)
58
+ else
59
+ $log.warn "Unsupported Netflow version v#{header.version}"
60
+ return
61
+ end
62
+
63
+ flowset.records.each do |record|
64
+ if flowset.version == 5
65
+ event = {}
66
+
67
+ # FIXME Probably not doing this right WRT JRuby?
68
+ #
69
+ # The flowset header gives us the UTC epoch seconds along with
70
+ # residual nanoseconds so we can set @timestamp to that easily
71
+ time = flowset.unix_sec
72
+
73
+ # Copy some of the pertinent fields in the header to the event
74
+ ['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records'].each do |f|
75
+ event[f] = flowset[f]
76
+ end
77
+
78
+ # Create fields in the event from each field in the flow record
79
+ record.each_pair do |k,v|
80
+ case k.to_s
81
+ when /_switched$/
82
+ # The flow record sets the first and last times to the device
83
+ # uptime in milliseconds. Given the actual uptime is provided
84
+ # in the flowset header along with the epoch seconds we can
85
+ # convert these into absolute times
86
+ millis = flowset.uptime - v
87
+ seconds = flowset.unix_sec - (millis / 1000)
88
+ micros = (flowset.unix_nsec / 1000) - (millis % 1000)
89
+ if micros < 0
90
+ seconds--
91
+ micros += 1000000
92
+ end
93
+
94
+ # FIXME Again, probably doing this wrong WRT JRuby?
95
+ event[k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
96
+ else
97
+ event[k.to_s] = v
98
+ end
99
+ end
100
+
101
+ yield time, event
102
+ elsif flowset.version == 9
103
+ case record.flowset_id
104
+ when 0
105
+ # Template flowset
106
+ record.flowset_data.templates.each do |template|
107
+ catch (:field) do
108
+ fields = []
109
+ template.fields.each do |field|
110
+ entry = netflow_field_for(field.field_type, field.field_length)
111
+ if !entry
112
+ throw :field
113
+ end
114
+ fields += entry
115
+ end
116
+ # We get this far, we have a list of fields
117
+ #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
118
+ key = "#{flowset.source_id}|#{template.template_id}"
119
+ @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
120
+ # Purge any expired templates
121
+ @templates.cleanup!
122
+ end
123
+ end
124
+ when 1
125
+ # Options template flowset
126
+ record.flowset_data.templates.each do |template|
127
+ catch (:field) do
128
+ fields = []
129
+ template.option_fields.each do |field|
130
+ entry = netflow_field_for(field.field_type, field.field_length)
131
+ if ! entry
132
+ throw :field
133
+ end
134
+ fields += entry
135
+ end
136
+ # We get this far, we have a list of fields
137
+ #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
138
+ key = "#{flowset.source_id}|#{template.template_id}"
139
+ @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
140
+ # Purge any expired templates
141
+ @templates.cleanup!
142
+ end
143
+ end
144
+ when 256..65535
145
+ # Data flowset
146
+ #key = "#{flowset.source_id}|#{event["source"]}|#{record.flowset_id}"
147
+ key = "#{flowset.source_id}|#{record.flowset_id}"
148
+ template = @templates[key]
149
+ if ! template
150
+ #$log.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
151
+ $log.warn("No matching template for flow id #{record.flowset_id}")
152
+ next
153
+ end
154
+
155
+ length = record.flowset_length - 4
156
+
157
+ # Template shouldn't be longer than the record and there should
158
+ # be at most 3 padding bytes
159
+ if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
160
+ $log.warn("Template length doesn't fit cleanly into flowset", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
161
+ next
162
+ end
163
+
164
+ array = BinData::Array.new(:type => template, :initial_length => length / template.num_bytes)
165
+
166
+ records = array.read(record.flowset_data)
167
+ records.each do |r|
168
+ time = flowset.unix_sec
169
+ event = {}
170
+
171
+ # Fewer fields in the v9 header
172
+ ['version', 'flow_seq_num'].each do |f|
173
+ event[f] = flowset[f]
174
+ end
175
+
176
+ event['flowset_id'] = record.flowset_id
177
+
178
+ r.each_pair do |k,v|
179
+ case k.to_s
180
+ when /_switched$/
181
+ millis = flowset.uptime - v
182
+ seconds = flowset.unix_sec - (millis / 1000)
183
+ # v9 did away with the nanosecs field
184
+ micros = 1000000 - (millis % 1000)
185
+ event[k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
186
+ else
187
+ event[k.to_s] = v
188
+ end
189
+ end
190
+
191
+ yield time, event
192
+ end
193
+ else
194
+ $log.warn("Unsupported flowset id #{record.flowset_id}")
195
+ end
196
+ end
197
+ end
198
+ end
199
+
200
+ private
201
+
202
+ def uint_field(length, default)
203
+ # If length is 4, return :uint32, etc. and use default if length is 0
204
+ ("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
205
+ end
206
+
207
+ def netflow_field_for(type, length)
208
+ if @fields.include?(type)
209
+ field = @fields[type]
210
+ if field.is_a?(Array)
211
+
212
+ if field[0].is_a?(Integer)
213
+ field[0] = uint_field(length, field[0])
214
+ end
215
+
216
+ # Small bit of fixup for skip or string field types where the length
217
+ # is dynamic
218
+ case field[0]
219
+ when :skip
220
+ field += [nil, {:length => length}]
221
+ when :string
222
+ field += [{:length => length, :trim_padding => true}]
223
+ end
224
+
225
+ [field]
226
+ else
227
+ $log.warn("Definition should be an array", :field => field)
228
+ nil
229
+ end
230
+ else
231
+ $log.warn("Unsupported field", :type => type, :length => length)
232
+ nil
233
+ end
234
+ end
235
+
236
+ class IP4Addr < BinData::Primitive
237
+ endian :big
238
+ uint32 :storage
239
+
240
+ def set(val)
241
+ ip = IPAddr.new(val)
242
+ if ! ip.ipv4?
243
+ raise ArgumentError, "invalid IPv4 address '#{val}'"
244
+ end
245
+ self.storage = ip.to_i
246
+ end
247
+
248
+ def get
249
+ IPAddr.new_ntoh([self.storage].pack('N')).to_s
250
+ end
251
+ end
252
+
253
+ class IP6Addr < BinData::Primitive
254
+ endian :big
255
+ uint128 :storage
256
+
257
+ def set(val)
258
+ ip = IPAddr.new(val)
259
+ if ! ip.ipv6?
260
+ raise ArgumentError, "invalid IPv6 address `#{val}'"
261
+ end
262
+ self.storage = ip.to_i
263
+ end
264
+
265
+ def get
266
+ IPAddr.new_ntoh((0..7).map { |i|
267
+ (self.storage >> (112 - 16 * i)) & 0xffff
268
+ }.pack('n8')).to_s
269
+ end
270
+ end
271
+
272
+ class MacAddr < BinData::Primitive
273
+ array :bytes, :type => :uint8, :initial_length => 6
274
+
275
+ def set(val)
276
+ ints = val.split(/:/).collect { |int| int.to_i(16) }
277
+ self.bytes = ints
278
+ end
279
+
280
+ def get
281
+ self.bytes.collect { |byte| byte.to_s(16) }.join(":")
282
+ end
283
+ end
284
+
285
+ class Header < BinData::Record
286
+ endian :big
287
+ uint16 :version
288
+ end
289
+
290
+ class Netflow5PDU < BinData::Record
291
+ endian :big
292
+ uint16 :version
293
+ uint16 :flow_records
294
+ uint32 :uptime
295
+ uint32 :unix_sec
296
+ uint32 :unix_nsec
297
+ uint32 :flow_seq_num
298
+ uint8 :engine_type
299
+ uint8 :engine_id
300
+ bit2 :sampling_algorithm
301
+ bit14 :sampling_interval
302
+ array :records, :initial_length => :flow_records do
303
+ ip4_addr :ipv4_src_addr
304
+ ip4_addr :ipv4_dst_addr
305
+ ip4_addr :ipv4_next_hop
306
+ uint16 :input_snmp
307
+ uint16 :output_snmp
308
+ uint32 :in_pkts
309
+ uint32 :in_bytes
310
+ uint32 :first_switched
311
+ uint32 :last_switched
312
+ uint16 :l4_src_port
313
+ uint16 :l4_dst_port
314
+ skip :length => 1
315
+ uint8 :tcp_flags # Split up the TCP flags maybe?
316
+ uint8 :protocol
317
+ uint8 :src_tos
318
+ uint16 :src_as
319
+ uint16 :dst_as
320
+ uint8 :src_mask
321
+ uint8 :dst_mask
322
+ skip :length => 2
323
+ end
324
+ end
325
+
326
+ class TemplateFlowset < BinData::Record
327
+ endian :big
328
+ array :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
329
+ uint16 :template_id
330
+ uint16 :field_count
331
+ array :fields, :initial_length => :field_count do
332
+ uint16 :field_type
333
+ uint16 :field_length
334
+ end
335
+ end
336
+ end
337
+
338
+ class OptionFlowset < BinData::Record
339
+ endian :big
340
+ array :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
341
+ uint16 :template_id
342
+ uint16 :scope_length
343
+ uint16 :option_length
344
+ array :scope_fields, :initial_length => lambda { scope_length / 4 } do
345
+ uint16 :field_type
346
+ uint16 :field_length
347
+ end
348
+ array :option_fields, :initial_length => lambda { option_length / 4 } do
349
+ uint16 :field_type
350
+ uint16 :field_length
351
+ end
352
+ end
353
+ skip :length => lambda { templates.length.odd? ? 2 : 0 }
354
+ end
355
+
356
+ class Netflow9PDU < BinData::Record
357
+ endian :big
358
+ uint16 :version
359
+ uint16 :flow_records
360
+ uint32 :uptime
361
+ uint32 :unix_sec
362
+ uint32 :flow_seq_num
363
+ uint32 :source_id
364
+ array :records, :read_until => :eof do
365
+ uint16 :flowset_id
366
+ uint16 :flowset_length
367
+ choice :flowset_data, :selection => :flowset_id do
368
+ template_flowset 0
369
+ option_flowset 1
370
+ string :default, :read_length => lambda { flowset_length - 4 }
371
+ end
372
+ end
373
+ end
374
+
375
+ # https://gist.github.com/joshaven/184837
376
+ class Vash < Hash
377
+ def initialize(constructor = {})
378
+ @register ||= {}
379
+ if constructor.is_a?(Hash)
380
+ super()
381
+ merge(constructor)
382
+ else
383
+ super(constructor)
384
+ end
385
+ end
386
+
387
+ alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
388
+ alias_method :regular_reader, :[] unless method_defined?(:regular_reader)
389
+
390
+ def [](key)
391
+ sterilize(key)
392
+ clear(key) if expired?(key)
393
+ regular_reader(key)
394
+ end
395
+
396
+ def []=(key, *args)
397
+ if args.length == 2
398
+ value, ttl = args[1], args[0]
399
+ elsif args.length == 1
400
+ value, ttl = args[0], 60
401
+ else
402
+ raise ArgumentError, "Wrong number of arguments, expected 2 or 3, received: #{args.length+1}\n"+
403
+ "Example Usage: volatile_hash[:key]=value OR volatile_hash[:key, ttl]=value"
404
+ end
405
+ sterilize(key)
406
+ ttl(key, ttl)
407
+ regular_writer(key, value)
408
+ end
409
+
410
+ def merge(hsh)
411
+ hsh.map {|key,value| self[sterile(key)] = hsh[key]}
412
+ self
413
+ end
414
+
415
+ def cleanup!
416
+ now = Time.now.to_i
417
+ @register.map {|k,v| clear(k) if v < now}
418
+ end
419
+
420
+ def clear(key)
421
+ sterilize(key)
422
+ @register.delete key
423
+ self.delete key
424
+ end
425
+
426
+ private
427
+
428
+ def expired?(key)
429
+ Time.now.to_i > @register[key].to_i
430
+ end
431
+
432
+ def ttl(key, secs=60)
433
+ @register[key] = Time.now.to_i + secs.to_i
434
+ end
435
+
436
+ def sterile(key)
437
+ String === key ? key.chomp('!').chomp('=') : key.to_s.chomp('!').chomp('=').to_sym
438
+ end
439
+
440
+ def sterilize(key)
441
+ key = sterile(key)
442
+ end
443
+ end
444
+ end
445
+
446
+ register_template('netflow', self)
447
+ end
448
+ end
metadata ADDED
@@ -0,0 +1,94 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-netflow
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Masahiro Nakagawa
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-06-20 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 0.10.17
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 0.10.17
27
+ - !ruby/object:Gem::Dependency
28
+ name: bindata
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '2.1'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '2.1'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: 0.9.2
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: 0.9.2
55
+ description: Netflow plugin for Fluentd
56
+ email: repeatedly@gmail.com
57
+ executables: []
58
+ extensions: []
59
+ extra_rdoc_files: []
60
+ files:
61
+ - ".travis.yml"
62
+ - Gemfile
63
+ - README.md
64
+ - Rakefile
65
+ - VERSION
66
+ - fluent-plugin-netflow.gemspec
67
+ - lib/fluent/plugin/in_netflow.rb
68
+ - lib/fluent/plugin/netflow.yaml
69
+ - lib/fluent/plugin/parser_netflow.rb
70
+ homepage: https://github.com/repeatedly/fluent-plugin-netflow
71
+ licenses:
72
+ - Apache License (2.0)
73
+ metadata: {}
74
+ post_install_message:
75
+ rdoc_options: []
76
+ require_paths:
77
+ - lib
78
+ required_ruby_version: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ required_rubygems_version: !ruby/object:Gem::Requirement
84
+ requirements:
85
+ - - ">="
86
+ - !ruby/object:Gem::Version
87
+ version: '0'
88
+ requirements: []
89
+ rubyforge_project:
90
+ rubygems_version: 2.2.2
91
+ signing_key:
92
+ specification_version: 4
93
+ summary: Netflow plugin for Fluentd
94
+ test_files: []