logstash-codec-netflow 3.1.4 → 3.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/CONTRIBUTORS +1 -0
- data/lib/logstash/codecs/netflow.rb +112 -87
- data/lib/logstash/codecs/netflow/ipfix.yaml +664 -0
- data/lib/logstash/codecs/netflow/util.rb +44 -0
- data/logstash-codec-netflow.gemspec +1 -1
- data/spec/codecs/ipfix_test_netscaler_data.dat +0 -0
- data/spec/codecs/ipfix_test_netscaler_tpl.dat +0 -0
- data/spec/codecs/netflow_spec.rb +303 -0
- metadata +42 -36
@@ -51,6 +51,36 @@ class MacAddr < BinData::Primitive
|
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
|
+
class VarSkip < BinData::Primitive
|
55
|
+
endian :big
|
56
|
+
uint8 :length_1
|
57
|
+
uint16 :length_2, :onlyif => lambda { length_1 == 255 }
|
58
|
+
skip :length => lambda { (length_1 == 255) ? length_2 : length_1 }
|
59
|
+
|
60
|
+
def get
|
61
|
+
""
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
class VarString < BinData::Primitive
|
66
|
+
endian :big
|
67
|
+
uint8 :length_1
|
68
|
+
uint16 :length_2, :onlyif => lambda { length_1 == 255 }
|
69
|
+
string :data, :trim_padding => true, :length => lambda { (length_1 == 255) ? length_2 : length_1 }
|
70
|
+
|
71
|
+
def set(val)
|
72
|
+
self.data = val
|
73
|
+
end
|
74
|
+
|
75
|
+
def get
|
76
|
+
self.data
|
77
|
+
end
|
78
|
+
|
79
|
+
def snapshot
|
80
|
+
super.encode("ASCII-8BIT", "UTF-8", invalid: :replace, undef: :replace)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
54
84
|
class ACLIdASA < BinData::Primitive
|
55
85
|
array :bytes, :type => :uint8, :initial_length => 12
|
56
86
|
|
@@ -78,6 +108,20 @@ class Forwarding_Status < BinData::Record
|
|
78
108
|
bit6 :reason
|
79
109
|
end
|
80
110
|
|
111
|
+
class OctetArray < BinData::Primitive
|
112
|
+
# arg_processor :octetarray
|
113
|
+
mandatory_parameter :initial_length
|
114
|
+
array :bytes, :type => :uint8, :initial_length => :initial_length
|
115
|
+
|
116
|
+
def set(val)
|
117
|
+
self.bytes = val.scan(/../).collect { |hex| hex.to_i(16)}
|
118
|
+
end
|
119
|
+
|
120
|
+
def get
|
121
|
+
self.bytes.collect { |byte| byte.value.to_s(16).rjust(2,'0') }.join
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
81
125
|
class Header < BinData::Record
|
82
126
|
endian :big
|
83
127
|
uint16 :version
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-netflow'
|
4
|
-
s.version = '3.
|
4
|
+
s.version = '3.2.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows."
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
Binary file
|
Binary file
|
data/spec/codecs/netflow_spec.rb
CHANGED
@@ -966,5 +966,308 @@ describe LogStash::Codecs::Netflow do
|
|
966
966
|
|
967
967
|
end
|
968
968
|
|
969
|
+
context "IPFIX Netscaler with variable length fields" do
|
970
|
+
let(:data) do
|
971
|
+
# this ipfix raw data was produced by a Netscaler appliance and captured with wireshark
|
972
|
+
# select packet bytes were then exported and sort of Pseudonymized to protect corp data
|
973
|
+
data = []
|
974
|
+
data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_tpl.dat"), :mode => "rb")
|
975
|
+
data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
|
976
|
+
end
|
977
|
+
|
978
|
+
let(:json_events) do
|
979
|
+
events = []
|
980
|
+
events << <<-END
|
981
|
+
{
|
982
|
+
"@timestamp": "2016-11-11T12:09:19.000Z",
|
983
|
+
"netflow": {
|
984
|
+
"netscalerHttpReqUserAgent": "Mozilla/5.0 (Commodore 64; kobo.com) Gecko/20100101 Firefox/75.0",
|
985
|
+
"destinationTransportPort": 443,
|
986
|
+
"netscalerHttpReqCookie": "beer=123456789abcdefghijklmnopqrstuvw; AnotherCookie=1234567890abcdefghijklmnopqr; Shameless.Plug=Thankyou.Rakuten.Kobo.Inc.For.Allowing.me.time.to.work.on.this.and.contribute.back.to.the.community; Padding=aaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbccccccccccccccddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffgggggggggggggggggggggggghhhhhhhhhhhhhhhhhiiiiiiiiiiiiiiiiiiiiiijjjjjjjjjjjjjjjjjjjjjjjjkkkkkkkkkkkkkkkkkklllllllllllllllmmmmmmmmmm; more=less; GJquote=There.is.no.spoon; GarrySays=Nice!!; LastPadding=aaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbcccccccccccccccccccdddddddddddeeeeeeee",
|
987
|
+
"flowEndMicroseconds": "503894-10-15T08:48:16.972Z",
|
988
|
+
"netscalerHttpReqUrl": "/aa/bb/ccccc/ddddddddddddddddddddddddd",
|
989
|
+
"sourceIPv4Address": "192.168.0.1",
|
990
|
+
"netscalerHttpReqMethod": "GET",
|
991
|
+
"netscalerHttpReqHost": "www.kobo.com",
|
992
|
+
"egressInterface": 2147483651,
|
993
|
+
"octetDeltaCount": 1541,
|
994
|
+
"netscalerAppNameAppId": 240189440,
|
995
|
+
"sourceTransportPort": 51053,
|
996
|
+
"flowId": 14460661,
|
997
|
+
"netscalerHttpReqAuthorization": "",
|
998
|
+
"netscalerHttpDomainName": "www.kobo.com",
|
999
|
+
"netscalerAaaUsername": "",
|
1000
|
+
"netscalerHttpContentType": "",
|
1001
|
+
"destinationIPv4Address": "10.0.0.1",
|
1002
|
+
"observationPointId": 167954698,
|
1003
|
+
"netscalerHttpReqVia": "1.1 akamai.net(ghost) (AkamaiGHost)",
|
1004
|
+
"netscalerConnectionId": 14460661,
|
1005
|
+
"tcpControlBits": 24,
|
1006
|
+
"flowStartMicroseconds": "503894-10-15T08:48:16.972Z",
|
1007
|
+
"ingressInterface": 8,
|
1008
|
+
"version": 10,
|
1009
|
+
"packetDeltaCount": 2,
|
1010
|
+
"netscalerUnknown330": 0,
|
1011
|
+
"netscalerConnectionChainID": "00e0ed1c9ca80300efb42558596b0800",
|
1012
|
+
"ipVersion": 4,
|
1013
|
+
"protocolIdentifier": 6,
|
1014
|
+
"netscalerHttpResForwLB": 0,
|
1015
|
+
"netscalerHttpReqReferer": "http://www.kobo.com/is-the-best-ebook-company-in-the-world",
|
1016
|
+
"exportingProcessId": 3,
|
1017
|
+
"netscalerAppUnitNameAppId": 239927296,
|
1018
|
+
"netscalerFlowFlags": 84025344,
|
1019
|
+
"netscalerTransactionId": 1068114985,
|
1020
|
+
"netscalerHttpResForwFB": 0,
|
1021
|
+
"netscalerConnectionChainHopCount": 1,
|
1022
|
+
"netscalerHttpReqXForwardedFor": "11.222.33.255"
|
1023
|
+
},
|
1024
|
+
"@version": "1"
|
1025
|
+
}
|
1026
|
+
END
|
1027
|
+
end
|
1028
|
+
|
1029
|
+
it "should decode raw data" do
|
1030
|
+
expect(decode.size).to eq(3)
|
1031
|
+
expect(decode[0].get("[netflow][version]")).to eq(10)
|
1032
|
+
expect(decode[0].get("[netflow][sourceIPv4Address]")).to eq('192.168.0.1')
|
1033
|
+
expect(decode[0].get("[netflow][destinationIPv4Address]")).to eq('10.0.0.1')
|
1034
|
+
expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('503894-10-15T08:48:16.970Z')
|
1035
|
+
expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
|
1036
|
+
expect(decode[1].get("[netflow][version]")).to eq(10)
|
1037
|
+
expect(decode[1].get("[netflow][flowId]")).to eq(14460662)
|
1038
|
+
expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
|
1039
|
+
expect(decode[1].get("[netflow][netscalerFlowFlags]")).to eq(1157636096)
|
1040
|
+
expect(decode[1].get("[netflow][netscalerRoundTripTime]")).to eq(83)
|
1041
|
+
expect(decode[2].get("[netflow][version]")).to eq(10)
|
1042
|
+
expect(decode[2].get("[netflow][netscalerAppUnitNameAppId]")).to eq(239927296)
|
1043
|
+
expect(decode[2].get("[netflow][netscalerHttpReqXForwardedFor]")).to eq('11.222.33.255')
|
1044
|
+
end
|
1045
|
+
|
1046
|
+
it "should decode variable length fields" do
|
1047
|
+
expect(decode[2].get("[netflow][netscalerHttpReqUrl]")).to eq('/aa/bb/ccccc/ddddddddddddddddddddddddd')
|
1048
|
+
expect(decode[2].get("[netflow][netscalerHttpReqHost]")).to eq('www.kobo.com')
|
1049
|
+
expect(decode[2].get("[netflow][netscalerHttpReqUserAgent]")).to eq('Mozilla/5.0 (Commodore 64; kobo.com) Gecko/20100101 Firefox/75.0')
|
1050
|
+
expect(decode[2].get("[netflow][netscalerHttpReqVia]")).to eq('1.1 akamai.net(ghost) (AkamaiGHost)')
|
1051
|
+
end
|
1052
|
+
|
1053
|
+
it "should decode fields with more than 255 chars" do
|
1054
|
+
expect(decode[2].get("[netflow][netscalerHttpReqCookie]")).to eq('beer=123456789abcdefghijklmnopqrstuvw; AnotherCookie=1234567890abcdefghijklmnopqr; Shameless.Plug=Thankyou.Rakuten.Kobo.Inc.For.Allowing.me.time.to.work.on.this.and.contribute.back.to.the.community; Padding=aaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbccccccccccccccddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffgggggggggggggggggggggggghhhhhhhhhhhhhhhhhiiiiiiiiiiiiiiiiiiiiiijjjjjjjjjjjjjjjjjjjjjjjjkkkkkkkkkkkkkkkkkklllllllllllllllmmmmmmmmmm; more=less; GJquote=There.is.no.spoon; GarrySays=Nice!!; LastPadding=aaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbcccccccccccccccccccdddddddddddeeeeeeee')
|
1055
|
+
end
|
1056
|
+
|
1057
|
+
it "should decode octetarray data" do
|
1058
|
+
expect(decode[0].get("[netflow][netscalerConnectionChainID]")).to eq('00e0ed1c9ca80300efb4255884850600')
|
1059
|
+
end
|
1060
|
+
|
1061
|
+
it "should serialize to json" do
|
1062
|
+
expect(JSON.parse(decode[2].to_json)).to eq(JSON.parse(json_events[0]))
|
1063
|
+
end
|
1064
|
+
end
|
1065
|
+
end
|
1066
|
+
|
1067
|
+
describe LogStash::Codecs::Netflow, 'missing templates, no template caching configured' do
|
1068
|
+
subject do
|
1069
|
+
LogStash::Codecs::Netflow.new.tap do |codec|
|
1070
|
+
expect{codec.register}.not_to raise_error
|
1071
|
+
end
|
1072
|
+
end
|
1073
|
+
|
1074
|
+
let(:logger) { double("logger") }
|
1075
|
+
|
1076
|
+
before :each do
|
1077
|
+
allow(LogStash::Codecs::Netflow).to receive(:logger).and_return(logger)
|
1078
|
+
allow(logger).to receive(:debug) {}
|
1079
|
+
allow(logger).to receive(:warn) {}
|
1080
|
+
end
|
1081
|
+
|
1082
|
+
let(:decode) do
|
1083
|
+
[].tap do |events|
|
1084
|
+
data.each { |packet| subject.decode(packet){|event| events << event}}
|
1085
|
+
end
|
1086
|
+
end
|
1087
|
+
|
1088
|
+
context "IPFIX Netscaler with variable length fields, missing templates" do
|
1089
|
+
let(:data) do
|
1090
|
+
data = []
|
1091
|
+
data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
|
1092
|
+
end
|
1093
|
+
|
1094
|
+
it "can not / should not decode any data" do
|
1095
|
+
expect(decode.size).to eq(0)
|
1096
|
+
expect{decode[0].get("[netflow][version]")}.to raise_error(NoMethodError, /undefined method .get. for nil:NilClass/)
|
1097
|
+
expect{JSON.parse(decode[0].to_json)}.to raise_error(JSON::ParserError)
|
1098
|
+
end
|
1099
|
+
|
1100
|
+
it "should report missing templates" do
|
1101
|
+
expect(logger).to receive(:warn).with(/No matching template for flow id/)
|
1102
|
+
decode[0]
|
1103
|
+
end
|
1104
|
+
end
|
1105
|
+
end
|
1106
|
+
|
1107
|
+
# New subject with config, ordered testing since we need caching before data processing
|
1108
|
+
describe LogStash::Codecs::Netflow, 'configured with template caching', :order => :defined do
|
1109
|
+
context "IPFIX Netscaler with variable length fields" do
|
1110
|
+
subject do
|
1111
|
+
LogStash::Codecs::Netflow.new(cache_config).tap do |codec|
|
1112
|
+
expect{codec.register}.not_to raise_error
|
1113
|
+
end
|
1114
|
+
end
|
1115
|
+
|
1116
|
+
let(:tmp_dir) { ENV["TMP"] || ENV["TMPDIR"] || ENV["TEMP"] || "/tmp" }
|
1117
|
+
|
1118
|
+
let(:cache_config) do
|
1119
|
+
{ "cache_save_path" => tmp_dir }
|
1120
|
+
end
|
1121
|
+
|
1122
|
+
let(:data) do
|
1123
|
+
# this ipfix raw data was produced by a Netscaler appliance and captured with wireshark
|
1124
|
+
# select packet bytes were then exported and sort of Pseudonymized to protect corp data
|
1125
|
+
data = []
|
1126
|
+
data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
|
1127
|
+
end
|
1128
|
+
|
1129
|
+
let(:templates) do
|
1130
|
+
templates = []
|
1131
|
+
templates << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_tpl.dat"), :mode => "rb")
|
1132
|
+
end
|
1133
|
+
|
1134
|
+
let(:cache) do
|
1135
|
+
[].tap do |events|
|
1136
|
+
templates.each { |packet| subject.decode(packet){|event| events << event}}
|
1137
|
+
end
|
1138
|
+
end
|
1139
|
+
|
1140
|
+
let(:decode) do
|
1141
|
+
[].tap do |events|
|
1142
|
+
data.each { |packet| subject.decode(packet){|event| events << event}}
|
1143
|
+
end
|
1144
|
+
end
|
1145
|
+
|
1146
|
+
let(:cached_templates) do
|
1147
|
+
cached_templates = <<-END
|
1148
|
+
{
|
1149
|
+
"0|256": [
|
1150
|
+
["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
|
1151
|
+
["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
|
1152
|
+
["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
|
1153
|
+
["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
|
1154
|
+
["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"]],
|
1155
|
+
"0|257": [
|
1156
|
+
["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
|
1157
|
+
["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
|
1158
|
+
["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
|
1159
|
+
["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","netscalerRoundTripTime"],["uint32","egressInterface"],
|
1160
|
+
["uint32","ingressInterface"],["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],
|
1161
|
+
["uint16","netscalerUnknown329"],["uint16","netscalerUnknown331"],["uint32","netscalerUnknown332"]],
|
1162
|
+
"0|258": [
|
1163
|
+
["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
|
1164
|
+
["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
|
1165
|
+
["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
|
1166
|
+
["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
|
1167
|
+
["uint32","netscalerAppNameAppId"],["uint32","netscalerAppUnitNameAppId"],["uint64","netscalerHttpResForwFB"],["uint64","netscalerHttpResForwLB"],
|
1168
|
+
["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"],
|
1169
|
+
["VarString","netscalerAaaUsername"],["VarString","netscalerHttpReqUrl"],["VarString","netscalerHttpReqCookie"],["VarString","netscalerHttpReqReferer"],
|
1170
|
+
["VarString","netscalerHttpReqMethod"],["VarString","netscalerHttpReqHost"],["VarString","netscalerHttpReqUserAgent"],["VarString","netscalerHttpContentType"],
|
1171
|
+
["VarString","netscalerHttpReqAuthorization"],["VarString","netscalerHttpReqVia"],["VarString","netscalerHttpReqXForwardedFor"],["VarString","netscalerHttpDomainName"]],
|
1172
|
+
"0|259": [
|
1173
|
+
["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
|
1174
|
+
["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip6_addr","sourceIPv6Address"],["ip6_addr","destinationIPv6Address"],
|
1175
|
+
["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
|
1176
|
+
["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
|
1177
|
+
["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"]],
|
1178
|
+
"0|260": [
|
1179
|
+
["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
|
1180
|
+
["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip6_addr","sourceIPv6Address"],["ip6_addr","destinationIPv6Address"],
|
1181
|
+
["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
|
1182
|
+
["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","netscalerRoundTripTime"],["uint32","egressInterface"],
|
1183
|
+
["uint32","ingressInterface"],["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],
|
1184
|
+
["uint16","netscalerUnknown329"],["uint16","netscalerUnknown331"],["uint32","netscalerUnknown332"]],
|
1185
|
+
"0|261": [
|
1186
|
+
["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
|
1187
|
+
["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip6_addr","sourceIPv6Address"],["ip6_addr","destinationIPv6Address"],
|
1188
|
+
["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
|
1189
|
+
["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
|
1190
|
+
["uint32","netscalerAppNameAppId"],["uint32","netscalerAppUnitNameAppId"],["uint64","netscalerHttpResForwFB"],["uint64","netscalerHttpResForwLB"],
|
1191
|
+
["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"],
|
1192
|
+
["uint32","netscalerCacheRedirClientConnectionCoreID"],["uint32","netscalerCacheRedirClientConnectionTransactionID"],["VarString","netscalerAaaUsername"],
|
1193
|
+
["VarString","netscalerHttpReqUrl"],["VarString","netscalerHttpReqCookie"],["VarString","netscalerHttpReqReferer"],["VarString","netscalerHttpReqMethod"],
|
1194
|
+
["VarString","netscalerHttpReqHost"],["VarString","netscalerHttpReqUserAgent"],["VarString","netscalerHttpContentType"],["VarString","netscalerHttpReqAuthorization"],
|
1195
|
+
["VarString","netscalerHttpReqVia"],["VarString","netscalerHttpReqXForwardedFor"],["VarString","netscalerHttpDomainName"]],
|
1196
|
+
"0|262": [
|
1197
|
+
["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
|
1198
|
+
["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
|
1199
|
+
["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
|
1200
|
+
["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
|
1201
|
+
["uint16","netscalerHttpRspStatus"],["uint64","netscalerHttpRspLen"],["uint64","netscalerServerTTFB"],["uint64","netscalerServerTTLB"],
|
1202
|
+
["uint32","netscalerAppNameAppId"],["uint32","netscalerMainPageId"],["uint32","netscalerMainPageCoreId"],["uint64","netscalerHttpReqRcvFB"],
|
1203
|
+
["uint64","netscalerHttpReqForwFB"],["uint64","netscalerHttpResRcvFB"],["uint64","netscalerHttpReqRcvLB"],["uint64","netscalerHttpReqForwLB"],
|
1204
|
+
["uint64","netscalerHttpResRcvLB"],["uint32","netscalerClientRTT"],["uint16","netscalerUnknown330"],["uint32","netscalerUnknown347"],["VarString","netscalerAaaUsername"],
|
1205
|
+
["VarString","netscalerHttpContentType"],["VarString","netscalerHttpResLocation"],["VarString","netscalerHttpResSetCookie"],["VarString","netscalerHttpResSetCookie2"]]
|
1206
|
+
}
|
1207
|
+
END
|
1208
|
+
end
|
969
1209
|
|
1210
|
+
it "should cache templates" do
|
1211
|
+
expect(cache.size).to eq(0)
|
1212
|
+
expect(JSON.parse(File.read("#{tmp_dir}/ipfix_templates.cache"))).to eq(JSON.parse(cached_templates))
|
1213
|
+
end
|
1214
|
+
|
1215
|
+
it "should decode raw data based on cached templates" do
|
1216
|
+
expect(decode.size).to eq(3)
|
1217
|
+
expect(decode[0].get("[netflow][version]")).to eq(10)
|
1218
|
+
expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('503894-10-15T08:48:16.970Z')
|
1219
|
+
expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
|
1220
|
+
expect(decode[1].get("[netflow][version]")).to eq(10)
|
1221
|
+
expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
|
1222
|
+
expect(decode[1].get("[netflow][netscalerFlowFlags]")).to eq(1157636096)
|
1223
|
+
expect(decode[2].get("[netflow][version]")).to eq(10)
|
1224
|
+
expect(decode[2].get("[netflow][netscalerAppUnitNameAppId]")).to eq(239927296)
|
1225
|
+
expect(decode[2].get("[netflow][netscalerHttpReqXForwardedFor]")).to eq('11.222.33.255')
|
1226
|
+
FileUtils.rm_rf(tmp_dir)
|
1227
|
+
end
|
1228
|
+
end
|
1229
|
+
end
|
1230
|
+
|
1231
|
+
describe LogStash::Codecs::Netflow, 'configured with include_flowset_id for ipfix' do
|
1232
|
+
subject do
|
1233
|
+
LogStash::Codecs::Netflow.new(include_flowset_id_config).tap do |codec|
|
1234
|
+
expect{codec.register}.not_to raise_error
|
1235
|
+
end
|
1236
|
+
end
|
1237
|
+
|
1238
|
+
let(:include_flowset_id_config) do
|
1239
|
+
{ "include_flowset_id" => true }
|
1240
|
+
end
|
1241
|
+
|
1242
|
+
let(:decode) do
|
1243
|
+
[].tap do |events|
|
1244
|
+
data.each { |packet| subject.decode(packet){|event| events << event}}
|
1245
|
+
end
|
1246
|
+
end
|
1247
|
+
|
1248
|
+
let(:data) do
|
1249
|
+
# this ipfix raw data was produced by a Netscaler appliance and captured with wireshark
|
1250
|
+
# select packet bytes were then exported and sort of Pseudonymized to protect corp data
|
1251
|
+
data = []
|
1252
|
+
data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_tpl.dat"), :mode => "rb")
|
1253
|
+
data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
|
1254
|
+
end
|
1255
|
+
|
1256
|
+
it "should decode raw data" do
|
1257
|
+
expect(decode.size).to eq(3)
|
1258
|
+
expect(decode[0].get("[netflow][version]")).to eq(10)
|
1259
|
+
expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('503894-10-15T08:48:16.970Z')
|
1260
|
+
expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
|
1261
|
+
expect(decode[1].get("[netflow][version]")).to eq(10)
|
1262
|
+
expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
|
1263
|
+
expect(decode[1].get("[netflow][netscalerFlowFlags]")).to eq(1157636096)
|
1264
|
+
expect(decode[2].get("[netflow][version]")).to eq(10)
|
1265
|
+
expect(decode[2].get("[netflow][netscalerAppUnitNameAppId]")).to eq(239927296)
|
1266
|
+
end
|
1267
|
+
|
1268
|
+
it "should include flowset_id" do
|
1269
|
+
expect(decode[0].get("[netflow][flowset_id]")).to eq(258)
|
1270
|
+
expect(decode[1].get("[netflow][flowset_id]")).to eq(257)
|
1271
|
+
expect(decode[2].get("[netflow][flowset_id]")).to eq(258)
|
1272
|
+
end
|
970
1273
|
end
|
metadata
CHANGED
@@ -1,81 +1,76 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-netflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
date: 2016-12-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
+
name: logstash-core-plugin-api
|
14
15
|
requirement: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
|
-
- -
|
17
|
+
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
18
19
|
version: '1.60'
|
19
|
-
- -
|
20
|
+
- - <=
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: '2.99'
|
22
|
-
name: logstash-core-plugin-api
|
23
|
-
prerelease: false
|
24
23
|
type: :runtime
|
24
|
+
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
|
-
- -
|
27
|
+
- - '>='
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '1.60'
|
30
|
-
- -
|
30
|
+
- - <=
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2.99'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: bindata
|
34
35
|
requirement: !ruby/object:Gem::Requirement
|
35
36
|
requirements:
|
36
|
-
- -
|
37
|
+
- - '>='
|
37
38
|
- !ruby/object:Gem::Version
|
38
39
|
version: 1.5.0
|
39
|
-
name: bindata
|
40
|
-
prerelease: false
|
41
40
|
type: :runtime
|
41
|
+
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
|
-
- -
|
44
|
+
- - '>='
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: 1.5.0
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
|
+
name: logstash-devutils
|
48
49
|
requirement: !ruby/object:Gem::Requirement
|
49
50
|
requirements:
|
50
|
-
- -
|
51
|
+
- - '>='
|
51
52
|
- !ruby/object:Gem::Version
|
52
53
|
version: 1.0.0
|
53
|
-
name: logstash-devutils
|
54
|
-
prerelease: false
|
55
54
|
type: :development
|
55
|
+
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
|
-
- -
|
58
|
+
- - '>='
|
59
59
|
- !ruby/object:Gem::Version
|
60
60
|
version: 1.0.0
|
61
|
-
description: This gem is a Logstash plugin required to be installed on top of the
|
61
|
+
description: This gem is a Logstash plugin required to be installed on top of the
|
62
|
+
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
63
|
+
gem is not a stand-alone program
|
62
64
|
email: info@elastic.co
|
63
65
|
executables: []
|
64
66
|
extensions: []
|
65
67
|
extra_rdoc_files: []
|
66
68
|
files:
|
67
|
-
- CHANGELOG.md
|
68
|
-
- CONTRIBUTORS
|
69
|
-
- Gemfile
|
70
|
-
- LICENSE
|
71
|
-
- NOTICE.TXT
|
72
|
-
- README.md
|
73
|
-
- lib/logstash/codecs/netflow.rb
|
74
69
|
- lib/logstash/codecs/netflow/iana2yaml.rb
|
75
|
-
- lib/logstash/codecs/netflow/ipfix.yaml
|
76
70
|
- lib/logstash/codecs/netflow/netflow.yaml
|
71
|
+
- lib/logstash/codecs/netflow/ipfix.yaml
|
77
72
|
- lib/logstash/codecs/netflow/util.rb
|
78
|
-
- logstash
|
73
|
+
- lib/logstash/codecs/netflow.rb
|
79
74
|
- spec/codecs/ipfix.dat
|
80
75
|
- spec/codecs/ipfix_test_openbsd_pflow_data.dat
|
81
76
|
- spec/codecs/ipfix_test_openbsd_pflow_tpl.dat
|
@@ -95,35 +90,44 @@ files:
|
|
95
90
|
- spec/codecs/netflow9_test_nprobe_data.dat
|
96
91
|
- spec/codecs/netflow9_test_nprobe_tpl.dat
|
97
92
|
- spec/codecs/netflow9_test_softflowd_tpl_data.dat
|
93
|
+
- spec/codecs/netflow9_test_valid01.dat
|
94
|
+
- spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
|
98
95
|
- spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
|
99
96
|
- spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
|
100
|
-
- spec/codecs/
|
101
|
-
- spec/codecs/
|
97
|
+
- spec/codecs/ipfix_test_netscaler_data.dat
|
98
|
+
- spec/codecs/ipfix_test_netscaler_tpl.dat
|
102
99
|
- spec/codecs/netflow_spec.rb
|
100
|
+
- logstash-codec-netflow.gemspec
|
101
|
+
- README.md
|
102
|
+
- CHANGELOG.md
|
103
|
+
- CONTRIBUTORS
|
104
|
+
- Gemfile
|
105
|
+
- LICENSE
|
106
|
+
- NOTICE.TXT
|
103
107
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
104
108
|
licenses:
|
105
109
|
- Apache License (2.0)
|
106
110
|
metadata:
|
107
111
|
logstash_plugin: 'true'
|
108
112
|
logstash_group: codec
|
109
|
-
post_install_message:
|
113
|
+
post_install_message:
|
110
114
|
rdoc_options: []
|
111
115
|
require_paths:
|
112
116
|
- lib
|
113
117
|
required_ruby_version: !ruby/object:Gem::Requirement
|
114
118
|
requirements:
|
115
|
-
- -
|
119
|
+
- - '>='
|
116
120
|
- !ruby/object:Gem::Version
|
117
121
|
version: '0'
|
118
122
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
119
123
|
requirements:
|
120
|
-
- -
|
124
|
+
- - '>='
|
121
125
|
- !ruby/object:Gem::Version
|
122
126
|
version: '0'
|
123
127
|
requirements: []
|
124
|
-
rubyforge_project:
|
125
|
-
rubygems_version: 2.
|
126
|
-
signing_key:
|
128
|
+
rubyforge_project:
|
129
|
+
rubygems_version: 2.0.14
|
130
|
+
signing_key:
|
127
131
|
specification_version: 4
|
128
132
|
summary: The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows.
|
129
133
|
test_files:
|
@@ -146,8 +150,10 @@ test_files:
|
|
146
150
|
- spec/codecs/netflow9_test_nprobe_data.dat
|
147
151
|
- spec/codecs/netflow9_test_nprobe_tpl.dat
|
148
152
|
- spec/codecs/netflow9_test_softflowd_tpl_data.dat
|
153
|
+
- spec/codecs/netflow9_test_valid01.dat
|
154
|
+
- spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
|
149
155
|
- spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
|
150
156
|
- spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
|
151
|
-
- spec/codecs/
|
152
|
-
- spec/codecs/
|
157
|
+
- spec/codecs/ipfix_test_netscaler_data.dat
|
158
|
+
- spec/codecs/ipfix_test_netscaler_tpl.dat
|
153
159
|
- spec/codecs/netflow_spec.rb
|