logstash-codec-netflow 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/.gitignore +3 -0
- data/Gemfile +4 -0
- data/LICENSE +13 -0
- data/Rakefile +6 -0
- data/lib/logstash/codecs/netflow.rb +263 -0
- data/lib/logstash/codecs/netflow/netflow.yaml +215 -0
- data/lib/logstash/codecs/netflow/util.rb +212 -0
- data/logstash-codec-netflow.gemspec +26 -0
- data/rakelib/publish.rake +9 -0
- data/rakelib/vendor.rake +169 -0
- data/spec/codecs/netflow_spec.rb +1 -0
- metadata +93 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
MTc3NTE4OWRjM2RiYzc5MDhlNjViMTQ1MzZiNWFiMmIyODcwOGYwZg==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
MjEzYTdhMDVkNzcwYjE1MWI5ZmY4MzFjZGFhYjQ1YmEzYWE1NzAyYQ==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
YTg3N2Q1NTVmMzcwOWY4YWYxM2MxMWI1YTIyYmY1NjQzOTQxMDgwYzZkYjAy
|
10
|
+
Nzk3ZjI5NTJjMzA4MjAyMWQ0YjFjNmU0NDViOTI1MmEyNWI4ZGMyNjJkYzdj
|
11
|
+
ZWU1MzhkMzAxZjM2Y2EyZmVlNzUwYmQzZjkwYjdmMDMzYmZkZTE=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
OWJkODUxNDkxZDJhYTQ2YWY5Yzk3MDljMTBjNDkzOGJkZDZlZGFhY2E4NWE3
|
14
|
+
MGNkYTk0ZDMxODc3NGZmNTg2YTVjNjRkYzk3NTYyNzJhOWI4OTE2ZjM3MGRh
|
15
|
+
ZTk0MWIyNWI2Nzg5NjQ4NjQ5OGRkZDk0ZjIzNGJmNWMyN2Y0Y2Q=
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/Rakefile
ADDED
@@ -0,0 +1,263 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/filters/base"
|
3
|
+
require "logstash/namespace"
|
4
|
+
require "logstash/timestamp"
|
5
|
+
|
6
|
+
# The "netflow" codec is for decoding Netflow v5/v9 flows.
|
7
|
+
class LogStash::Codecs::Netflow < LogStash::Codecs::Base
|
8
|
+
config_name "netflow"
|
9
|
+
milestone 1
|
10
|
+
|
11
|
+
# Netflow v9 template cache TTL (minutes)
|
12
|
+
config :cache_ttl, :validate => :number, :default => 4000
|
13
|
+
|
14
|
+
# Specify into what field you want the Netflow data.
|
15
|
+
config :target, :validate => :string, :default => "netflow"
|
16
|
+
|
17
|
+
# Specify which Netflow versions you will accept.
|
18
|
+
config :versions, :validate => :array, :default => [5, 9]
|
19
|
+
|
20
|
+
# Override YAML file containing Netflow field definitions
|
21
|
+
#
|
22
|
+
# Each Netflow field is defined like so:
|
23
|
+
#
|
24
|
+
# ---
|
25
|
+
# id:
|
26
|
+
# - default length in bytes
|
27
|
+
# - :name
|
28
|
+
# id:
|
29
|
+
# - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
|
30
|
+
# - :name
|
31
|
+
# id:
|
32
|
+
# - :skip
|
33
|
+
#
|
34
|
+
# See <https://github.com/logstash/logstash/tree/v%VERSION%/lib/logstash/codecs/netflow/netflow.yaml> for the base set.
|
35
|
+
config :definitions, :validate => :path
|
36
|
+
|
37
|
+
public
|
38
|
+
def initialize(params={})
|
39
|
+
super(params)
|
40
|
+
@threadsafe = false
|
41
|
+
end
|
42
|
+
|
43
|
+
public
|
44
|
+
def register
|
45
|
+
require "logstash/codecs/netflow/util"
|
46
|
+
@templates = Vash.new()
|
47
|
+
|
48
|
+
# Path to default Netflow v9 field definitions
|
49
|
+
filename = ::File.expand_path('netflow/netflow.yaml', ::File.dirname(__FILE__))
|
50
|
+
|
51
|
+
begin
|
52
|
+
@fields = YAML.load_file(filename)
|
53
|
+
rescue Exception => e
|
54
|
+
raise "#{self.class.name}: Bad syntax in definitions file #{filename}"
|
55
|
+
end
|
56
|
+
|
57
|
+
# Allow the user to augment/override/rename the supported Netflow fields
|
58
|
+
if @definitions
|
59
|
+
raise "#{self.class.name}: definitions file #{@definitions} does not exists" unless File.exists?(@definitions)
|
60
|
+
begin
|
61
|
+
@fields.merge!(YAML.load_file(@definitions))
|
62
|
+
rescue Exception => e
|
63
|
+
raise "#{self.class.name}: Bad syntax in definitions file #{@definitions}"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end # def register
|
67
|
+
|
68
|
+
public
|
69
|
+
def decode(payload, &block)
|
70
|
+
header = Header.read(payload)
|
71
|
+
|
72
|
+
unless @versions.include?(header.version)
|
73
|
+
@logger.warn("Ignoring Netflow version v#{header.version}")
|
74
|
+
return
|
75
|
+
end
|
76
|
+
|
77
|
+
if header.version == 5
|
78
|
+
flowset = Netflow5PDU.read(payload)
|
79
|
+
elsif header.version == 9
|
80
|
+
flowset = Netflow9PDU.read(payload)
|
81
|
+
else
|
82
|
+
@logger.warn("Unsupported Netflow version v#{header.version}")
|
83
|
+
return
|
84
|
+
end
|
85
|
+
|
86
|
+
flowset.records.each do |record|
|
87
|
+
if flowset.version == 5
|
88
|
+
event = LogStash::Event.new
|
89
|
+
|
90
|
+
# FIXME Probably not doing this right WRT JRuby?
|
91
|
+
#
|
92
|
+
# The flowset header gives us the UTC epoch seconds along with
|
93
|
+
# residual nanoseconds so we can set @timestamp to that easily
|
94
|
+
event.timestamp = LogStash::Timestamp.at(flowset.unix_sec, flowset.unix_nsec / 1000)
|
95
|
+
event[@target] = {}
|
96
|
+
|
97
|
+
# Copy some of the pertinent fields in the header to the event
|
98
|
+
['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records'].each do |f|
|
99
|
+
event[@target][f] = flowset[f]
|
100
|
+
end
|
101
|
+
|
102
|
+
# Create fields in the event from each field in the flow record
|
103
|
+
record.each_pair do |k,v|
|
104
|
+
case k.to_s
|
105
|
+
when /_switched$/
|
106
|
+
# The flow record sets the first and last times to the device
|
107
|
+
# uptime in milliseconds. Given the actual uptime is provided
|
108
|
+
# in the flowset header along with the epoch seconds we can
|
109
|
+
# convert these into absolute times
|
110
|
+
millis = flowset.uptime - v
|
111
|
+
seconds = flowset.unix_sec - (millis / 1000)
|
112
|
+
micros = (flowset.unix_nsec / 1000) - (millis % 1000)
|
113
|
+
if micros < 0
|
114
|
+
seconds--
|
115
|
+
micros += 1000000
|
116
|
+
end
|
117
|
+
# FIXME Again, probably doing this wrong WRT JRuby?
|
118
|
+
event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
|
119
|
+
else
|
120
|
+
event[@target][k.to_s] = v
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
yield event
|
125
|
+
elsif flowset.version == 9
|
126
|
+
case record.flowset_id
|
127
|
+
when 0
|
128
|
+
# Template flowset
|
129
|
+
record.flowset_data.templates.each do |template|
|
130
|
+
catch (:field) do
|
131
|
+
fields = []
|
132
|
+
template.fields.each do |field|
|
133
|
+
entry = netflow_field_for(field.field_type, field.field_length)
|
134
|
+
if ! entry
|
135
|
+
throw :field
|
136
|
+
end
|
137
|
+
fields += entry
|
138
|
+
end
|
139
|
+
# We get this far, we have a list of fields
|
140
|
+
#key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
|
141
|
+
key = "#{flowset.source_id}|#{template.template_id}"
|
142
|
+
@templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
|
143
|
+
# Purge any expired templates
|
144
|
+
@templates.cleanup!
|
145
|
+
end
|
146
|
+
end
|
147
|
+
when 1
|
148
|
+
# Options template flowset
|
149
|
+
record.flowset_data.templates.each do |template|
|
150
|
+
catch (:field) do
|
151
|
+
fields = []
|
152
|
+
template.option_fields.each do |field|
|
153
|
+
entry = netflow_field_for(field.field_type, field.field_length)
|
154
|
+
if ! entry
|
155
|
+
throw :field
|
156
|
+
end
|
157
|
+
fields += entry
|
158
|
+
end
|
159
|
+
# We get this far, we have a list of fields
|
160
|
+
#key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
|
161
|
+
key = "#{flowset.source_id}|#{template.template_id}"
|
162
|
+
@templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
|
163
|
+
# Purge any expired templates
|
164
|
+
@templates.cleanup!
|
165
|
+
end
|
166
|
+
end
|
167
|
+
when 256..65535
|
168
|
+
# Data flowset
|
169
|
+
#key = "#{flowset.source_id}|#{event["source"]}|#{record.flowset_id}"
|
170
|
+
key = "#{flowset.source_id}|#{record.flowset_id}"
|
171
|
+
template = @templates[key]
|
172
|
+
|
173
|
+
if ! template
|
174
|
+
#@logger.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
|
175
|
+
@logger.warn("No matching template for flow id #{record.flowset_id}")
|
176
|
+
next
|
177
|
+
end
|
178
|
+
|
179
|
+
length = record.flowset_length - 4
|
180
|
+
|
181
|
+
# Template shouldn't be longer than the record and there should
|
182
|
+
# be at most 3 padding bytes
|
183
|
+
if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
|
184
|
+
@logger.warn("Template length doesn't fit cleanly into flowset", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
|
185
|
+
next
|
186
|
+
end
|
187
|
+
|
188
|
+
array = BinData::Array.new(:type => template, :initial_length => length / template.num_bytes)
|
189
|
+
|
190
|
+
records = array.read(record.flowset_data)
|
191
|
+
|
192
|
+
records.each do |r|
|
193
|
+
event = LogStash::Event.new(
|
194
|
+
LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(flowset.unix_sec),
|
195
|
+
@target => {}
|
196
|
+
)
|
197
|
+
|
198
|
+
# Fewer fields in the v9 header
|
199
|
+
['version', 'flow_seq_num'].each do |f|
|
200
|
+
event[@target][f] = flowset[f]
|
201
|
+
end
|
202
|
+
|
203
|
+
event[@target]['flowset_id'] = record.flowset_id
|
204
|
+
|
205
|
+
r.each_pair do |k,v|
|
206
|
+
case k.to_s
|
207
|
+
when /_switched$/
|
208
|
+
millis = flowset.uptime - v
|
209
|
+
seconds = flowset.unix_sec - (millis / 1000)
|
210
|
+
# v9 did away with the nanosecs field
|
211
|
+
micros = 1000000 - (millis % 1000)
|
212
|
+
event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
|
213
|
+
else
|
214
|
+
event[@target][k.to_s] = v
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
yield event
|
219
|
+
end
|
220
|
+
else
|
221
|
+
@logger.warn("Unsupported flowset id #{record.flowset_id}")
|
222
|
+
end
|
223
|
+
end
|
224
|
+
end
|
225
|
+
end # def filter
|
226
|
+
|
227
|
+
private
|
228
|
+
def uint_field(length, default)
|
229
|
+
# If length is 4, return :uint32, etc. and use default if length is 0
|
230
|
+
("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
|
231
|
+
end # def uint_field
|
232
|
+
|
233
|
+
private
|
234
|
+
def netflow_field_for(type, length)
|
235
|
+
if @fields.include?(type)
|
236
|
+
field = @fields[type]
|
237
|
+
if field.is_a?(Array)
|
238
|
+
|
239
|
+
if field[0].is_a?(Integer)
|
240
|
+
field[0] = uint_field(length, field[0])
|
241
|
+
end
|
242
|
+
|
243
|
+
# Small bit of fixup for skip or string field types where the length
|
244
|
+
# is dynamic
|
245
|
+
case field[0]
|
246
|
+
when :skip
|
247
|
+
field += [nil, {:length => length}]
|
248
|
+
when :string
|
249
|
+
field += [{:length => length, :trim_padding => true}]
|
250
|
+
end
|
251
|
+
|
252
|
+
@logger.debug("Definition complete", :field => field)
|
253
|
+
[field]
|
254
|
+
else
|
255
|
+
@logger.warn("Definition should be an array", :field => field)
|
256
|
+
nil
|
257
|
+
end
|
258
|
+
else
|
259
|
+
@logger.warn("Unsupported field", :type => type, :length => length)
|
260
|
+
nil
|
261
|
+
end
|
262
|
+
end # def netflow_field_for
|
263
|
+
end # class LogStash::Filters::Netflow
|
@@ -0,0 +1,215 @@
|
|
1
|
+
---
|
2
|
+
1:
|
3
|
+
- 4
|
4
|
+
- :in_bytes
|
5
|
+
2:
|
6
|
+
- 4
|
7
|
+
- :in_pkts
|
8
|
+
3:
|
9
|
+
- 4
|
10
|
+
- :flows
|
11
|
+
4:
|
12
|
+
- :uint8
|
13
|
+
- :protocol
|
14
|
+
5:
|
15
|
+
- :uint8
|
16
|
+
- :src_tos
|
17
|
+
6:
|
18
|
+
- :uint8
|
19
|
+
- :tcp_flags
|
20
|
+
7:
|
21
|
+
- :uint16
|
22
|
+
- :l4_src_port
|
23
|
+
8:
|
24
|
+
- :ip4_addr
|
25
|
+
- :ipv4_src_addr
|
26
|
+
9:
|
27
|
+
- :uint8
|
28
|
+
- :src_mask
|
29
|
+
10:
|
30
|
+
- 2
|
31
|
+
- :input_snmp
|
32
|
+
11:
|
33
|
+
- :uint16
|
34
|
+
- :l4_dst_port
|
35
|
+
12:
|
36
|
+
- :ip4_addr
|
37
|
+
- :ipv4_dst_addr
|
38
|
+
13:
|
39
|
+
- :uint8
|
40
|
+
- :dst_mask
|
41
|
+
14:
|
42
|
+
- 2
|
43
|
+
- :output_snmp
|
44
|
+
15:
|
45
|
+
- :ip4_addr
|
46
|
+
- :ipv4_next_hop
|
47
|
+
16:
|
48
|
+
- 2
|
49
|
+
- :src_as
|
50
|
+
17:
|
51
|
+
- 2
|
52
|
+
- :dst_as
|
53
|
+
18:
|
54
|
+
- :ip4_addr
|
55
|
+
- :bgp_ipv4_next_hop
|
56
|
+
19:
|
57
|
+
- 4
|
58
|
+
- :mul_dst_pkts
|
59
|
+
20:
|
60
|
+
- 4
|
61
|
+
- :mul_dst_bytes
|
62
|
+
21:
|
63
|
+
- :uint32
|
64
|
+
- :last_switched
|
65
|
+
22:
|
66
|
+
- :uint32
|
67
|
+
- :first_switched
|
68
|
+
23:
|
69
|
+
- 4
|
70
|
+
- :out_bytes
|
71
|
+
24:
|
72
|
+
- 4
|
73
|
+
- :out_pkts
|
74
|
+
25:
|
75
|
+
- :uint16
|
76
|
+
- :min_pkt_length
|
77
|
+
26:
|
78
|
+
- :uint16
|
79
|
+
- :max_pkt_length
|
80
|
+
27:
|
81
|
+
- :ip6_addr
|
82
|
+
- :ipv6_src_addr
|
83
|
+
28:
|
84
|
+
- :ip6_addr
|
85
|
+
- :ipv6_dst_addr
|
86
|
+
29:
|
87
|
+
- :uint8
|
88
|
+
- :ipv6_src_mask
|
89
|
+
30:
|
90
|
+
- :uint8
|
91
|
+
- :ipv6_dst_mask
|
92
|
+
31:
|
93
|
+
- :uint24
|
94
|
+
- :ipv6_flow_label
|
95
|
+
32:
|
96
|
+
- :uint16
|
97
|
+
- :icmp_type
|
98
|
+
33:
|
99
|
+
- :uint8
|
100
|
+
- :mul_igmp_type
|
101
|
+
34:
|
102
|
+
- :uint32
|
103
|
+
- :sampling_interval
|
104
|
+
35:
|
105
|
+
- :uint8
|
106
|
+
- :sampling_algorithm
|
107
|
+
36:
|
108
|
+
- :uint16
|
109
|
+
- :flow_active_timeout
|
110
|
+
37:
|
111
|
+
- :uint16
|
112
|
+
- :flow_inactive_timeout
|
113
|
+
38:
|
114
|
+
- :uint8
|
115
|
+
- :engine_type
|
116
|
+
39:
|
117
|
+
- :uint8
|
118
|
+
- :engine_id
|
119
|
+
40:
|
120
|
+
- 4
|
121
|
+
- :total_bytes_exp
|
122
|
+
41:
|
123
|
+
- 4
|
124
|
+
- :total_pkts_exp
|
125
|
+
42:
|
126
|
+
- 4
|
127
|
+
- :total_flows_exp
|
128
|
+
43:
|
129
|
+
- :skip
|
130
|
+
44:
|
131
|
+
- :ip4_addr
|
132
|
+
- :ipv4_src_prefix
|
133
|
+
45:
|
134
|
+
- :ip4_addr
|
135
|
+
- :ipv4_dst_prefix
|
136
|
+
46:
|
137
|
+
- :uint8
|
138
|
+
- :mpls_top_label_type
|
139
|
+
47:
|
140
|
+
- :uint32
|
141
|
+
- :mpls_top_label_ip_addr
|
142
|
+
48:
|
143
|
+
- 4
|
144
|
+
- :flow_sampler_id
|
145
|
+
49:
|
146
|
+
- :uint8
|
147
|
+
- :flow_sampler_mode
|
148
|
+
50:
|
149
|
+
- :uint32
|
150
|
+
- :flow_sampler_random_interval
|
151
|
+
51:
|
152
|
+
- :skip
|
153
|
+
52:
|
154
|
+
- :uint8
|
155
|
+
- :min_ttl
|
156
|
+
53:
|
157
|
+
- :uint8
|
158
|
+
- :max_ttl
|
159
|
+
54:
|
160
|
+
- :uint16
|
161
|
+
- :ipv4_ident
|
162
|
+
55:
|
163
|
+
- :uint8
|
164
|
+
- :dst_tos
|
165
|
+
56:
|
166
|
+
- :mac_addr
|
167
|
+
- :in_src_max
|
168
|
+
57:
|
169
|
+
- :mac_addr
|
170
|
+
- :out_dst_max
|
171
|
+
58:
|
172
|
+
- :uint16
|
173
|
+
- :src_vlan
|
174
|
+
59:
|
175
|
+
- :uint16
|
176
|
+
- :dst_vlan
|
177
|
+
60:
|
178
|
+
- :uint8
|
179
|
+
- :ip_protocol_version
|
180
|
+
61:
|
181
|
+
- :uint8
|
182
|
+
- :direction
|
183
|
+
62:
|
184
|
+
- :ip6_addr
|
185
|
+
- :ipv6_next_hop
|
186
|
+
63:
|
187
|
+
- :ip6_addr
|
188
|
+
- :bgp_ipv6_next_hop
|
189
|
+
64:
|
190
|
+
- :uint32
|
191
|
+
- :ipv6_option_headers
|
192
|
+
64:
|
193
|
+
- :skip
|
194
|
+
65:
|
195
|
+
- :skip
|
196
|
+
66:
|
197
|
+
- :skip
|
198
|
+
67:
|
199
|
+
- :skip
|
200
|
+
68:
|
201
|
+
- :skip
|
202
|
+
69:
|
203
|
+
- :skip
|
204
|
+
80:
|
205
|
+
- :mac_addr
|
206
|
+
- :in_dst_mac
|
207
|
+
81:
|
208
|
+
- :mac_addr
|
209
|
+
- :out_src_mac
|
210
|
+
82:
|
211
|
+
- :string
|
212
|
+
- :if_name
|
213
|
+
83:
|
214
|
+
- :string
|
215
|
+
- :if_desc
|
@@ -0,0 +1,212 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "bindata"
|
3
|
+
require "ipaddr"
|
4
|
+
|
5
|
+
class IP4Addr < BinData::Primitive
|
6
|
+
endian :big
|
7
|
+
uint32 :storage
|
8
|
+
|
9
|
+
def set(val)
|
10
|
+
ip = IPAddr.new(val)
|
11
|
+
if ! ip.ipv4?
|
12
|
+
raise ArgumentError, "invalid IPv4 address '#{val}'"
|
13
|
+
end
|
14
|
+
self.storage = ip.to_i
|
15
|
+
end
|
16
|
+
|
17
|
+
def get
|
18
|
+
IPAddr.new_ntoh([self.storage].pack('N')).to_s
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
class IP6Addr < BinData::Primitive
|
23
|
+
endian :big
|
24
|
+
uint128 :storage
|
25
|
+
|
26
|
+
def set(val)
|
27
|
+
ip = IPAddr.new(val)
|
28
|
+
if ! ip.ipv6?
|
29
|
+
raise ArgumentError, "invalid IPv6 address `#{val}'"
|
30
|
+
end
|
31
|
+
self.storage = ip.to_i
|
32
|
+
end
|
33
|
+
|
34
|
+
def get
|
35
|
+
IPAddr.new_ntoh((0..7).map { |i|
|
36
|
+
(self.storage >> (112 - 16 * i)) & 0xffff
|
37
|
+
}.pack('n8')).to_s
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
class MacAddr < BinData::Primitive
|
42
|
+
array :bytes, :type => :uint8, :initial_length => 6
|
43
|
+
|
44
|
+
def set(val)
|
45
|
+
ints = val.split(/:/).collect { |int| int.to_i(16) }
|
46
|
+
self.bytes = ints
|
47
|
+
end
|
48
|
+
|
49
|
+
def get
|
50
|
+
self.bytes.collect { |byte| byte.to_s(16) }.join(":")
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
class Header < BinData::Record
|
55
|
+
endian :big
|
56
|
+
uint16 :version
|
57
|
+
end
|
58
|
+
|
59
|
+
class Netflow5PDU < BinData::Record
|
60
|
+
endian :big
|
61
|
+
uint16 :version
|
62
|
+
uint16 :flow_records
|
63
|
+
uint32 :uptime
|
64
|
+
uint32 :unix_sec
|
65
|
+
uint32 :unix_nsec
|
66
|
+
uint32 :flow_seq_num
|
67
|
+
uint8 :engine_type
|
68
|
+
uint8 :engine_id
|
69
|
+
bit2 :sampling_algorithm
|
70
|
+
bit14 :sampling_interval
|
71
|
+
array :records, :initial_length => :flow_records do
|
72
|
+
ip4_addr :ipv4_src_addr
|
73
|
+
ip4_addr :ipv4_dst_addr
|
74
|
+
ip4_addr :ipv4_next_hop
|
75
|
+
uint16 :input_snmp
|
76
|
+
uint16 :output_snmp
|
77
|
+
uint32 :in_pkts
|
78
|
+
uint32 :in_bytes
|
79
|
+
uint32 :first_switched
|
80
|
+
uint32 :last_switched
|
81
|
+
uint16 :l4_src_port
|
82
|
+
uint16 :l4_dst_port
|
83
|
+
skip :length => 1
|
84
|
+
uint8 :tcp_flags # Split up the TCP flags maybe?
|
85
|
+
uint8 :protocol
|
86
|
+
uint8 :src_tos
|
87
|
+
uint16 :src_as
|
88
|
+
uint16 :dst_as
|
89
|
+
uint8 :src_mask
|
90
|
+
uint8 :dst_mask
|
91
|
+
skip :length => 2
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
class TemplateFlowset < BinData::Record
|
96
|
+
endian :big
|
97
|
+
array :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
|
98
|
+
uint16 :template_id
|
99
|
+
uint16 :field_count
|
100
|
+
array :fields, :initial_length => :field_count do
|
101
|
+
uint16 :field_type
|
102
|
+
uint16 :field_length
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
class OptionFlowset < BinData::Record
|
108
|
+
endian :big
|
109
|
+
array :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
|
110
|
+
uint16 :template_id
|
111
|
+
uint16 :scope_length
|
112
|
+
uint16 :option_length
|
113
|
+
array :scope_fields, :initial_length => lambda { scope_length / 4 } do
|
114
|
+
uint16 :field_type
|
115
|
+
uint16 :field_length
|
116
|
+
end
|
117
|
+
array :option_fields, :initial_length => lambda { option_length / 4 } do
|
118
|
+
uint16 :field_type
|
119
|
+
uint16 :field_length
|
120
|
+
end
|
121
|
+
end
|
122
|
+
skip :length => lambda { templates.length.odd? ? 2 : 0 }
|
123
|
+
end
|
124
|
+
|
125
|
+
class Netflow9PDU < BinData::Record
|
126
|
+
endian :big
|
127
|
+
uint16 :version
|
128
|
+
uint16 :flow_records
|
129
|
+
uint32 :uptime
|
130
|
+
uint32 :unix_sec
|
131
|
+
uint32 :flow_seq_num
|
132
|
+
uint32 :source_id
|
133
|
+
array :records, :read_until => :eof do
|
134
|
+
uint16 :flowset_id
|
135
|
+
uint16 :flowset_length
|
136
|
+
choice :flowset_data, :selection => :flowset_id do
|
137
|
+
template_flowset 0
|
138
|
+
option_flowset 1
|
139
|
+
string :default, :read_length => lambda { flowset_length - 4 }
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
# https://gist.github.com/joshaven/184837
|
145
|
+
class Vash < Hash
|
146
|
+
def initialize(constructor = {})
|
147
|
+
@register ||= {}
|
148
|
+
if constructor.is_a?(Hash)
|
149
|
+
super()
|
150
|
+
merge(constructor)
|
151
|
+
else
|
152
|
+
super(constructor)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
|
157
|
+
alias_method :regular_reader, :[] unless method_defined?(:regular_reader)
|
158
|
+
|
159
|
+
def [](key)
|
160
|
+
sterilize(key)
|
161
|
+
clear(key) if expired?(key)
|
162
|
+
regular_reader(key)
|
163
|
+
end
|
164
|
+
|
165
|
+
def []=(key, *args)
|
166
|
+
if args.length == 2
|
167
|
+
value, ttl = args[1], args[0]
|
168
|
+
elsif args.length == 1
|
169
|
+
value, ttl = args[0], 60
|
170
|
+
else
|
171
|
+
raise ArgumentError, "Wrong number of arguments, expected 2 or 3, received: #{args.length+1}\n"+
|
172
|
+
"Example Usage: volatile_hash[:key]=value OR volatile_hash[:key, ttl]=value"
|
173
|
+
end
|
174
|
+
sterilize(key)
|
175
|
+
ttl(key, ttl)
|
176
|
+
regular_writer(key, value)
|
177
|
+
end
|
178
|
+
|
179
|
+
def merge(hsh)
|
180
|
+
hsh.map {|key,value| self[sterile(key)] = hsh[key]}
|
181
|
+
self
|
182
|
+
end
|
183
|
+
|
184
|
+
def cleanup!
|
185
|
+
now = Time.now.to_i
|
186
|
+
@register.map {|k,v| clear(k) if v < now}
|
187
|
+
end
|
188
|
+
|
189
|
+
def clear(key)
|
190
|
+
sterilize(key)
|
191
|
+
@register.delete key
|
192
|
+
self.delete key
|
193
|
+
end
|
194
|
+
|
195
|
+
private
|
196
|
+
def expired?(key)
|
197
|
+
Time.now.to_i > @register[key].to_i
|
198
|
+
end
|
199
|
+
|
200
|
+
def ttl(key, secs=60)
|
201
|
+
@register[key] = Time.now.to_i + secs.to_i
|
202
|
+
end
|
203
|
+
|
204
|
+
def sterile(key)
|
205
|
+
String === key ? key.chomp('!').chomp('=') : key.to_s.chomp('!').chomp('=').to_sym
|
206
|
+
end
|
207
|
+
|
208
|
+
def sterilize(key)
|
209
|
+
key = sterile(key)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
|
3
|
+
s.name = 'logstash-codec-netflow'
|
4
|
+
s.version = '0.1.0'
|
5
|
+
s.licenses = ['Apache License (2.0)']
|
6
|
+
s.summary = "The netflow codec is for decoding Netflow v5/v9 flows."
|
7
|
+
s.description = "The netflow codec is for decoding Netflow v5/v9 flows."
|
8
|
+
s.authors = ["Elasticsearch"]
|
9
|
+
s.email = 'richard.pijnenburg@elasticsearch.com'
|
10
|
+
s.homepage = "http://logstash.net/"
|
11
|
+
s.require_paths = ["lib"]
|
12
|
+
|
13
|
+
# Files
|
14
|
+
s.files = `git ls-files`.split($\)
|
15
|
+
|
16
|
+
# Tests
|
17
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
18
|
+
|
19
|
+
# Special flag to let us know this is actually a logstash plugin
|
20
|
+
s.metadata = { "logstash_plugin" => "true", "group" => "codec" }
|
21
|
+
|
22
|
+
# Gem dependencies
|
23
|
+
s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
|
24
|
+
s.add_runtime_dependency 'bindata', ['>= 1.5.0']
|
25
|
+
end
|
26
|
+
|
@@ -0,0 +1,9 @@
|
|
1
|
+
require "gem_publisher"
|
2
|
+
|
3
|
+
desc "Publish gem to RubyGems.org"
|
4
|
+
task :publish_gem do |t|
|
5
|
+
gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
|
6
|
+
gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
|
7
|
+
puts "Published #{gem}" if gem
|
8
|
+
end
|
9
|
+
|
data/rakelib/vendor.rake
ADDED
@@ -0,0 +1,169 @@
|
|
1
|
+
require "net/http"
|
2
|
+
require "uri"
|
3
|
+
require "digest/sha1"
|
4
|
+
|
5
|
+
def vendor(*args)
|
6
|
+
return File.join("vendor", *args)
|
7
|
+
end
|
8
|
+
|
9
|
+
directory "vendor/" => ["vendor"] do |task, args|
|
10
|
+
mkdir task.name
|
11
|
+
end
|
12
|
+
|
13
|
+
def fetch(url, sha1, output)
|
14
|
+
|
15
|
+
puts "Downloading #{url}"
|
16
|
+
actual_sha1 = download(url, output)
|
17
|
+
|
18
|
+
if actual_sha1 != sha1
|
19
|
+
fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
|
20
|
+
end
|
21
|
+
end # def fetch
|
22
|
+
|
23
|
+
def file_fetch(url, sha1)
|
24
|
+
filename = File.basename( URI(url).path )
|
25
|
+
output = "vendor/#{filename}"
|
26
|
+
task output => [ "vendor/" ] do
|
27
|
+
begin
|
28
|
+
actual_sha1 = file_sha1(output)
|
29
|
+
if actual_sha1 != sha1
|
30
|
+
fetch(url, sha1, output)
|
31
|
+
end
|
32
|
+
rescue Errno::ENOENT
|
33
|
+
fetch(url, sha1, output)
|
34
|
+
end
|
35
|
+
end.invoke
|
36
|
+
|
37
|
+
return output
|
38
|
+
end
|
39
|
+
|
40
|
+
def file_sha1(path)
|
41
|
+
digest = Digest::SHA1.new
|
42
|
+
fd = File.new(path, "r")
|
43
|
+
while true
|
44
|
+
begin
|
45
|
+
digest << fd.sysread(16384)
|
46
|
+
rescue EOFError
|
47
|
+
break
|
48
|
+
end
|
49
|
+
end
|
50
|
+
return digest.hexdigest
|
51
|
+
ensure
|
52
|
+
fd.close if fd
|
53
|
+
end
|
54
|
+
|
55
|
+
def download(url, output)
|
56
|
+
uri = URI(url)
|
57
|
+
digest = Digest::SHA1.new
|
58
|
+
tmp = "#{output}.tmp"
|
59
|
+
Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
|
60
|
+
request = Net::HTTP::Get.new(uri.path)
|
61
|
+
http.request(request) do |response|
|
62
|
+
fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
|
63
|
+
size = (response["content-length"].to_i || -1).to_f
|
64
|
+
count = 0
|
65
|
+
File.open(tmp, "w") do |fd|
|
66
|
+
response.read_body do |chunk|
|
67
|
+
fd.write(chunk)
|
68
|
+
digest << chunk
|
69
|
+
if size > 0 && $stdout.tty?
|
70
|
+
count += chunk.bytesize
|
71
|
+
$stdout.write(sprintf("\r%0.2f%%", count/size * 100))
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
$stdout.write("\r \r") if $stdout.tty?
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
File.rename(tmp, output)
|
80
|
+
|
81
|
+
return digest.hexdigest
|
82
|
+
rescue SocketError => e
|
83
|
+
puts "Failure while downloading #{url}: #{e}"
|
84
|
+
raise
|
85
|
+
ensure
|
86
|
+
File.unlink(tmp) if File.exist?(tmp)
|
87
|
+
end # def download
|
88
|
+
|
89
|
+
def untar(tarball, &block)
|
90
|
+
require "archive/tar/minitar"
|
91
|
+
tgz = Zlib::GzipReader.new(File.open(tarball))
|
92
|
+
# Pull out typesdb
|
93
|
+
tar = Archive::Tar::Minitar::Input.open(tgz)
|
94
|
+
tar.each do |entry|
|
95
|
+
path = block.call(entry)
|
96
|
+
next if path.nil?
|
97
|
+
parent = File.dirname(path)
|
98
|
+
|
99
|
+
mkdir_p parent unless File.directory?(parent)
|
100
|
+
|
101
|
+
# Skip this file if the output file is the same size
|
102
|
+
if entry.directory?
|
103
|
+
mkdir path unless File.directory?(path)
|
104
|
+
else
|
105
|
+
entry_mode = entry.instance_eval { @mode } & 0777
|
106
|
+
if File.exists?(path)
|
107
|
+
stat = File.stat(path)
|
108
|
+
# TODO(sissel): Submit a patch to archive-tar-minitar upstream to
|
109
|
+
# expose headers in the entry.
|
110
|
+
entry_size = entry.instance_eval { @size }
|
111
|
+
# If file sizes are same, skip writing.
|
112
|
+
next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
|
113
|
+
end
|
114
|
+
puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
|
115
|
+
File.open(path, "w") do |fd|
|
116
|
+
# eof? check lets us skip empty files. Necessary because the API provided by
|
117
|
+
# Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
|
118
|
+
# IO object. Something about empty files in this EntryStream causes
|
119
|
+
# IO.copy_stream to throw "can't convert nil into String" on JRuby
|
120
|
+
# TODO(sissel): File a bug about this.
|
121
|
+
while !entry.eof?
|
122
|
+
chunk = entry.read(16384)
|
123
|
+
fd.write(chunk)
|
124
|
+
end
|
125
|
+
#IO.copy_stream(entry, fd)
|
126
|
+
end
|
127
|
+
File.chmod(entry_mode, path)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
tar.close
|
131
|
+
File.unlink(tarball) if File.file?(tarball)
|
132
|
+
end # def untar
|
133
|
+
|
134
|
+
def ungz(file)
|
135
|
+
|
136
|
+
outpath = file.gsub('.gz', '')
|
137
|
+
tgz = Zlib::GzipReader.new(File.open(file))
|
138
|
+
begin
|
139
|
+
File.open(outpath, "w") do |out|
|
140
|
+
IO::copy_stream(tgz, out)
|
141
|
+
end
|
142
|
+
File.unlink(file)
|
143
|
+
rescue
|
144
|
+
File.unlink(outpath) if File.file?(outpath)
|
145
|
+
raise
|
146
|
+
end
|
147
|
+
tgz.close
|
148
|
+
end
|
149
|
+
|
150
|
+
desc "Process any vendor files required for this plugin"
|
151
|
+
task "vendor" do |task, args|
|
152
|
+
|
153
|
+
@files.each do |file|
|
154
|
+
download = file_fetch(file['url'], file['sha1'])
|
155
|
+
if download =~ /.tar.gz/
|
156
|
+
prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
|
157
|
+
untar(download) do |entry|
|
158
|
+
if !file['files'].nil?
|
159
|
+
next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
|
160
|
+
out = entry.full_name.split("/").last
|
161
|
+
end
|
162
|
+
File.join('vendor', out)
|
163
|
+
end
|
164
|
+
elsif download =~ /.gz/
|
165
|
+
ungz(download)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require 'spec_helper'
|
metadata
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logstash-codec-netflow
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Elasticsearch
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-11-05 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: logstash
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ! '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.4.0
|
20
|
+
- - <
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 2.0.0
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 1.4.0
|
30
|
+
- - <
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 2.0.0
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: bindata
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ! '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: !binary |-
|
40
|
+
MS41LjA=
|
41
|
+
type: :runtime
|
42
|
+
prerelease: false
|
43
|
+
version_requirements: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ! '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: !binary |-
|
48
|
+
MS41LjA=
|
49
|
+
description: The netflow codec is for decoding Netflow v5/v9 flows.
|
50
|
+
email: richard.pijnenburg@elasticsearch.com
|
51
|
+
executables: []
|
52
|
+
extensions: []
|
53
|
+
extra_rdoc_files: []
|
54
|
+
files:
|
55
|
+
- .gitignore
|
56
|
+
- Gemfile
|
57
|
+
- LICENSE
|
58
|
+
- Rakefile
|
59
|
+
- lib/logstash/codecs/netflow.rb
|
60
|
+
- lib/logstash/codecs/netflow/netflow.yaml
|
61
|
+
- lib/logstash/codecs/netflow/util.rb
|
62
|
+
- logstash-codec-netflow.gemspec
|
63
|
+
- rakelib/publish.rake
|
64
|
+
- rakelib/vendor.rake
|
65
|
+
- spec/codecs/netflow_spec.rb
|
66
|
+
homepage: http://logstash.net/
|
67
|
+
licenses:
|
68
|
+
- Apache License (2.0)
|
69
|
+
metadata:
|
70
|
+
logstash_plugin: 'true'
|
71
|
+
group: codec
|
72
|
+
post_install_message:
|
73
|
+
rdoc_options: []
|
74
|
+
require_paths:
|
75
|
+
- lib
|
76
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - ! '>='
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '0'
|
81
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
requirements: []
|
87
|
+
rubyforge_project:
|
88
|
+
rubygems_version: 2.4.1
|
89
|
+
signing_key:
|
90
|
+
specification_version: 4
|
91
|
+
summary: The netflow codec is for decoding Netflow v5/v9 flows.
|
92
|
+
test_files:
|
93
|
+
- spec/codecs/netflow_spec.rb
|