logstash-codec-netflow 3.12.0 → 3.13.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/CONTRIBUTORS +2 -0
- data/lib/logstash/codecs/netflow.rb +14 -2
- data/lib/logstash/codecs/netflow/ipfix.yaml +74 -1
- data/logstash-codec-netflow.gemspec +1 -1
- data/spec/codecs/ipfix_test_barracuda_extended_uniflow_data256.dat +0 -0
- data/spec/codecs/ipfix_test_barracuda_extended_uniflow_tpl256.dat +0 -0
- data/spec/codecs/netflow9_test_iptnetflow_reduced_size_encoding_tpldata260.dat +0 -0
- data/spec/codecs/netflow_spec.rb +120 -0
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2e7dc24e899f3afdf8980d5815727b8394cf9b69
|
4
|
+
data.tar.gz: 3fa3f494f7ad39af68d3b99e6c918b995cd8cae7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d81ead3c2fe61b83376d7a97fa7c9ded2039d42b2f95d7d80e39727bb60dd305a80367775d75f060accd7ec611e06f3c355ca75a3d8a76ae8535d72b4f24dddf
|
7
|
+
data.tar.gz: fe71e6a1e05f666a8b43c3de974b98e59f7a4663f12dcad0879cf5ca7b41f02740d5fe30bbfd3476c3c5bad76866ca06961afabcd5f14b4de6620b703f696b9d
|
data/CHANGELOG.md
CHANGED
data/CONTRIBUTORS
CHANGED
@@ -19,6 +19,7 @@ Contributors:
|
|
19
19
|
* Jordan Sissel (jordansissel)
|
20
20
|
* Jorrit Folmer (jorritfolmer)
|
21
21
|
* Keenan Tims (ktims)
|
22
|
+
* Magnus Kessler (kesslerm)
|
22
23
|
* Marian Craciunescu (marian-craciunescu)
|
23
24
|
* Matt Dainty (bodgit)
|
24
25
|
* Paul Warren (pwarren)
|
@@ -28,6 +29,7 @@ Contributors:
|
|
28
29
|
* Pulkit Agrawal (propulkit)
|
29
30
|
* Raju Nair (rajutech76)
|
30
31
|
* Richard Pijnenburg (electrical)
|
32
|
+
* Rob Cowart (robcowart)
|
31
33
|
* Salvador Ferrer (salva-ferrer)
|
32
34
|
* Vishal Solanki
|
33
35
|
* Will Rigby (wrigby)
|
@@ -480,9 +480,10 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
|
|
480
480
|
|
481
481
|
# Small bit of fixup for:
|
482
482
|
# - skip or string field types where the length is dynamic
|
483
|
-
# - uint(8|16|24|32} where we use the length as specified by the
|
483
|
+
# - uint(8|16|24|32|64} where we use the length as specified by the
|
484
484
|
# template instead of the YAML (e.g. ipv6_flow_label is 3 bytes in
|
485
|
-
# the YAML and Cisco doc, but Cisco ASR9k sends 4 bytes)
|
485
|
+
# the YAML and Cisco doc, but Cisco ASR9k sends 4 bytes).
|
486
|
+
# Another usecase is supporting reduced-size encoding as per RFC7011 6.2
|
486
487
|
# - application_id where we use the length as specified by the
|
487
488
|
# template and map it to custom types for handling.
|
488
489
|
#
|
@@ -490,10 +491,21 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
|
|
490
491
|
when :uint8
|
491
492
|
field[0] = uint_field(length, field[0])
|
492
493
|
when :uint16
|
494
|
+
if length>2
|
495
|
+
@logger.warn("Reduced-size encoding for uint16 is larger than uint16", :field => field, :length => length)
|
496
|
+
end
|
493
497
|
field[0] = uint_field(length, field[0])
|
494
498
|
when :uint24
|
495
499
|
field[0] = uint_field(length, field[0])
|
496
500
|
when :uint32
|
501
|
+
if length>4
|
502
|
+
@logger.warn("Reduced-size encoding for uint32 is larger than uint32", :field => field, :length => length)
|
503
|
+
end
|
504
|
+
field[0] = uint_field(length, field[0])
|
505
|
+
when :uint64
|
506
|
+
if length>8
|
507
|
+
@logger.warn("Reduced-size encoding for uint64 is larger than uint64", :field => field, :length => length)
|
508
|
+
end
|
497
509
|
field[0] = uint_field(length, field[0])
|
498
510
|
when :application_id
|
499
511
|
case length
|
@@ -3770,4 +3770,77 @@
|
|
3770
3770
|
47:
|
3771
3771
|
- :string
|
3772
3772
|
- :proceraTemplateName
|
3773
|
-
|
3773
|
+
10704:
|
3774
|
+
1:
|
3775
|
+
- :uint32
|
3776
|
+
- :Timestamp
|
3777
|
+
2:
|
3778
|
+
- :uint8
|
3779
|
+
- :LogOp
|
3780
|
+
3:
|
3781
|
+
- :uint8
|
3782
|
+
- TrafficType
|
3783
|
+
4:
|
3784
|
+
- :string
|
3785
|
+
- :FW_Rule
|
3786
|
+
5:
|
3787
|
+
- :string
|
3788
|
+
- :ServiceName
|
3789
|
+
6:
|
3790
|
+
- :uint32
|
3791
|
+
- :Reason
|
3792
|
+
7:
|
3793
|
+
- :string
|
3794
|
+
- :ReasonText
|
3795
|
+
8:
|
3796
|
+
- :ip4_addr
|
3797
|
+
- :BindIPv4Address
|
3798
|
+
9:
|
3799
|
+
- :uint16
|
3800
|
+
- :BindTransportPort
|
3801
|
+
10:
|
3802
|
+
- :ip4_addr
|
3803
|
+
- :ConnIPv4Address
|
3804
|
+
11:
|
3805
|
+
- :uint16
|
3806
|
+
- :ConnTransportPort
|
3807
|
+
12:
|
3808
|
+
- :uint32
|
3809
|
+
- :AuditCounter
|
3810
|
+
12326:
|
3811
|
+
1:
|
3812
|
+
- :uint32
|
3813
|
+
- :Timestamp
|
3814
|
+
2:
|
3815
|
+
- :uint8
|
3816
|
+
- :LogOp
|
3817
|
+
3:
|
3818
|
+
- :uint8
|
3819
|
+
- TrafficType
|
3820
|
+
4:
|
3821
|
+
- :string
|
3822
|
+
- :FW_Rule
|
3823
|
+
5:
|
3824
|
+
- :string
|
3825
|
+
- :ServiceName
|
3826
|
+
6:
|
3827
|
+
- :uint32
|
3828
|
+
- :Reason
|
3829
|
+
7:
|
3830
|
+
- :string
|
3831
|
+
- :ReasonText
|
3832
|
+
8:
|
3833
|
+
- :ip4_addr
|
3834
|
+
- :BindIPv4Address
|
3835
|
+
9:
|
3836
|
+
- :uint16
|
3837
|
+
- :BindTransportPort
|
3838
|
+
10:
|
3839
|
+
- :ip4_addr
|
3840
|
+
- :ConnIPv4Address
|
3841
|
+
11:
|
3842
|
+
- :uint16
|
3843
|
+
- :ConnTransportPort
|
3844
|
+
12:
|
3845
|
+
- :uint32
|
3846
|
+
- :AuditCounter
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-netflow'
|
4
|
-
s.version = '3.
|
4
|
+
s.version = '3.13.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads Netflow v5, Netflow v9 and IPFIX data"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
Binary file
|
Binary file
|
data/spec/codecs/netflow_spec.rb
CHANGED
@@ -905,6 +905,62 @@ describe LogStash::Codecs::Netflow do
|
|
905
905
|
|
906
906
|
end
|
907
907
|
|
908
|
+
context "Netflow 9 ipt_netflow reduced size encoding" do
|
909
|
+
let(:data) do
|
910
|
+
packets = []
|
911
|
+
packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_iptnetflow_reduced_size_encoding_tpldata260.dat"), :mode => "rb")
|
912
|
+
end
|
913
|
+
|
914
|
+
let(:json_events) do
|
915
|
+
events = []
|
916
|
+
events << <<-END
|
917
|
+
{
|
918
|
+
"netflow": {
|
919
|
+
"l4_src_port": 443,
|
920
|
+
"last_switched": "2018-02-18T05:46:54.999Z",
|
921
|
+
"ingressPhysicalInterface": 7,
|
922
|
+
"in_bytes": 187,
|
923
|
+
"tcpOptions": 2164260864,
|
924
|
+
"in_dst_mac": "00:1b:21:bc:24:dd",
|
925
|
+
"protocol": 6,
|
926
|
+
"output_snmp": 8,
|
927
|
+
"ethernetType": 2048,
|
928
|
+
"src_tos": 0,
|
929
|
+
"l4_dst_port": 38164,
|
930
|
+
"input_snmp": 7,
|
931
|
+
"version": 9,
|
932
|
+
"in_pkts": 3,
|
933
|
+
"flow_seq_num": 344481,
|
934
|
+
"ipv4_next_hop": "10.232.5.1",
|
935
|
+
"flowset_id": 260,
|
936
|
+
"first_switched": "2018-02-18T05:46:54.999Z",
|
937
|
+
"tcp_flags": 25,
|
938
|
+
"ipv4_dst_addr": "10.233.150.21",
|
939
|
+
"ipv4_src_addr": "2.17.140.47",
|
940
|
+
"in_src_mac": "90:e2:ba:23:09:fc",
|
941
|
+
"egressPhysicalInterface": 8
|
942
|
+
},
|
943
|
+
"@timestamp": "2018-02-18T05:47:09.000Z",
|
944
|
+
"@version": "1"
|
945
|
+
}
|
946
|
+
END
|
947
|
+
|
948
|
+
events.map{|event| event.gsub(/\s+/, "")}
|
949
|
+
end
|
950
|
+
|
951
|
+
it "should decode raw data" do
|
952
|
+
expect(decode.size).to eq(12)
|
953
|
+
expect(decode[11].get("[netflow][in_dst_mac]")).to eq("00:1b:21:bc:24:dd")
|
954
|
+
expect(decode[11].get("[netflow][ipv4_src_addr]")).to eq("2.17.140.47")
|
955
|
+
end
|
956
|
+
|
957
|
+
it "should serialize to json" do
|
958
|
+
expect(JSON.parse(decode[11].to_json)).to eq(JSON.parse(json_events[0]))
|
959
|
+
end
|
960
|
+
|
961
|
+
end
|
962
|
+
|
963
|
+
|
908
964
|
context "Netflow 9 IE150 IE151" do
|
909
965
|
let(:data) do
|
910
966
|
packets = []
|
@@ -1085,6 +1141,70 @@ describe LogStash::Codecs::Netflow do
|
|
1085
1141
|
|
1086
1142
|
end
|
1087
1143
|
|
1144
|
+
context "IPFIX Barracuda extended uniflow template 256" do
|
1145
|
+
let(:data) do
|
1146
|
+
packets = []
|
1147
|
+
packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_barracuda_extended_uniflow_tpl256.dat"), :mode => "rb")
|
1148
|
+
packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_barracuda_extended_uniflow_data256.dat"), :mode => "rb")
|
1149
|
+
end
|
1150
|
+
|
1151
|
+
let(:json_events) do
|
1152
|
+
events = []
|
1153
|
+
events << <<-END
|
1154
|
+
{
|
1155
|
+
"netflow": {
|
1156
|
+
"FW_Rule": "MTH:MTH-MC-to-Inet",
|
1157
|
+
"AuditCounter": 4157725,
|
1158
|
+
"sourceIPv4Address": "64.235.151.76",
|
1159
|
+
"version": 10,
|
1160
|
+
"sourceTransportPort": 443,
|
1161
|
+
"sourceMacAddress": "00:00:00:00:00:00",
|
1162
|
+
"ingressInterface": 3689,
|
1163
|
+
"flowEndSysUpTime": 1957197969,
|
1164
|
+
"octetTotalCount": 0,
|
1165
|
+
"ConnTransportPort": 443,
|
1166
|
+
"ConnIPv4Address": "64.235.151.76",
|
1167
|
+
"firewallEvent": 1,
|
1168
|
+
"protocolIdentifier": 6,
|
1169
|
+
"flowStartSysUpTime": 1957197969,
|
1170
|
+
"TrafficType": 0,
|
1171
|
+
"destinationTransportPort": 51917,
|
1172
|
+
"packetTotalCount": 0,
|
1173
|
+
"BindIPv4Address": "213.208.150.99",
|
1174
|
+
"Timestamp": 1524039407,
|
1175
|
+
"flowDurationMilliseconds": 0,
|
1176
|
+
"ServiceName": "https",
|
1177
|
+
"BindTransportPort": 64238,
|
1178
|
+
"octetDeltaCount": 0,
|
1179
|
+
"packetDeltaCount": 0,
|
1180
|
+
"destinationIPv4Address": "10.236.5.4",
|
1181
|
+
"LogOp": 1,
|
1182
|
+
"Reason": 0,
|
1183
|
+
"egressInterface": 35233,
|
1184
|
+
"ReasonText": "Normal Operation"
|
1185
|
+
},
|
1186
|
+
"@version": "1",
|
1187
|
+
"@timestamp": "2018-04-18T08:16:47.000Z"
|
1188
|
+
}
|
1189
|
+
END
|
1190
|
+
|
1191
|
+
events.map{|event| event.gsub(/\s+/, "")}
|
1192
|
+
events.map{|event| event.gsub(/NormalOperation/, "Normal Operation")}
|
1193
|
+
end
|
1194
|
+
|
1195
|
+
it "should decode raw data" do
|
1196
|
+
expect(decode.size).to eq(2)
|
1197
|
+
expect(decode[1].get("[netflow][FW_Rule]")).to eq("MTH:MTH-MC-to-Inet")
|
1198
|
+
expect(decode[1].get("[netflow][ReasonText]")).to eq("Normal Operation")
|
1199
|
+
expect(decode[1].get("[netflow][BindIPv4Address]")).to eq("213.208.150.99")
|
1200
|
+
end
|
1201
|
+
|
1202
|
+
it "should serialize to json" do
|
1203
|
+
expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[0]))
|
1204
|
+
end
|
1205
|
+
|
1206
|
+
end
|
1207
|
+
|
1088
1208
|
|
1089
1209
|
|
1090
1210
|
context "Netflow 9 Ubiquiti Edgerouter with MPLS labels" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-netflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.13.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-04-
|
11
|
+
date: 2018-04-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -81,6 +81,8 @@ files:
|
|
81
81
|
- spec/codecs/benchmarks/netflow_bench_cisco_asr.py
|
82
82
|
- spec/codecs/ipfix.dat
|
83
83
|
- spec/codecs/ipfix_test_barracuda_data256.dat
|
84
|
+
- spec/codecs/ipfix_test_barracuda_extended_uniflow_data256.dat
|
85
|
+
- spec/codecs/ipfix_test_barracuda_extended_uniflow_tpl256.dat
|
84
86
|
- spec/codecs/ipfix_test_barracuda_tpl.dat
|
85
87
|
- spec/codecs/ipfix_test_mikrotik_data258.dat
|
86
88
|
- spec/codecs/ipfix_test_mikrotik_data259.dat
|
@@ -138,6 +140,7 @@ files:
|
|
138
140
|
- spec/codecs/netflow9_test_huawei_netstream_data.dat
|
139
141
|
- spec/codecs/netflow9_test_huawei_netstream_tpl.dat
|
140
142
|
- spec/codecs/netflow9_test_invalid01.dat
|
143
|
+
- spec/codecs/netflow9_test_iptnetflow_reduced_size_encoding_tpldata260.dat
|
141
144
|
- spec/codecs/netflow9_test_juniper_srx_tplopt.dat
|
142
145
|
- spec/codecs/netflow9_test_macaddr_data.dat
|
143
146
|
- spec/codecs/netflow9_test_macaddr_tpl.dat
|
@@ -191,6 +194,8 @@ test_files:
|
|
191
194
|
- spec/codecs/benchmarks/netflow_bench_cisco_asr.py
|
192
195
|
- spec/codecs/ipfix.dat
|
193
196
|
- spec/codecs/ipfix_test_barracuda_data256.dat
|
197
|
+
- spec/codecs/ipfix_test_barracuda_extended_uniflow_data256.dat
|
198
|
+
- spec/codecs/ipfix_test_barracuda_extended_uniflow_tpl256.dat
|
194
199
|
- spec/codecs/ipfix_test_barracuda_tpl.dat
|
195
200
|
- spec/codecs/ipfix_test_mikrotik_data258.dat
|
196
201
|
- spec/codecs/ipfix_test_mikrotik_data259.dat
|
@@ -248,6 +253,7 @@ test_files:
|
|
248
253
|
- spec/codecs/netflow9_test_huawei_netstream_data.dat
|
249
254
|
- spec/codecs/netflow9_test_huawei_netstream_tpl.dat
|
250
255
|
- spec/codecs/netflow9_test_invalid01.dat
|
256
|
+
- spec/codecs/netflow9_test_iptnetflow_reduced_size_encoding_tpldata260.dat
|
251
257
|
- spec/codecs/netflow9_test_juniper_srx_tplopt.dat
|
252
258
|
- spec/codecs/netflow9_test_macaddr_data.dat
|
253
259
|
- spec/codecs/netflow9_test_macaddr_tpl.dat
|