logstash-codec-netflow 3.1.4 → 3.2.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -51,6 +51,36 @@ class MacAddr < BinData::Primitive
51
51
  end
52
52
  end
53
53
 
54
+ class VarSkip < BinData::Primitive
55
+ endian :big
56
+ uint8 :length_1
57
+ uint16 :length_2, :onlyif => lambda { length_1 == 255 }
58
+ skip :length => lambda { (length_1 == 255) ? length_2 : length_1 }
59
+
60
+ def get
61
+ ""
62
+ end
63
+ end
64
+
65
+ class VarString < BinData::Primitive
66
+ endian :big
67
+ uint8 :length_1
68
+ uint16 :length_2, :onlyif => lambda { length_1 == 255 }
69
+ string :data, :trim_padding => true, :length => lambda { (length_1 == 255) ? length_2 : length_1 }
70
+
71
+ def set(val)
72
+ self.data = val
73
+ end
74
+
75
+ def get
76
+ self.data
77
+ end
78
+
79
+ def snapshot
80
+ super.encode("ASCII-8BIT", "UTF-8", invalid: :replace, undef: :replace)
81
+ end
82
+ end
83
+
54
84
  class ACLIdASA < BinData::Primitive
55
85
  array :bytes, :type => :uint8, :initial_length => 12
56
86
 
@@ -78,6 +108,20 @@ class Forwarding_Status < BinData::Record
78
108
  bit6 :reason
79
109
  end
80
110
 
111
+ class OctetArray < BinData::Primitive
112
+ # arg_processor :octetarray
113
+ mandatory_parameter :initial_length
114
+ array :bytes, :type => :uint8, :initial_length => :initial_length
115
+
116
+ def set(val)
117
+ self.bytes = val.scan(/../).collect { |hex| hex.to_i(16)}
118
+ end
119
+
120
+ def get
121
+ self.bytes.collect { |byte| byte.value.to_s(16).rjust(2,'0') }.join
122
+ end
123
+ end
124
+
81
125
  class Header < BinData::Record
82
126
  endian :big
83
127
  uint16 :version
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-netflow'
4
- s.version = '3.1.4'
4
+ s.version = '3.2.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows."
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -966,5 +966,308 @@ describe LogStash::Codecs::Netflow do
966
966
 
967
967
  end
968
968
 
969
+ context "IPFIX Netscaler with variable length fields" do
970
+ let(:data) do
971
+ # this ipfix raw data was produced by a Netscaler appliance and captured with wireshark
972
+ # select packet bytes were then exported and sort of Pseudonymized to protect corp data
973
+ data = []
974
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_tpl.dat"), :mode => "rb")
975
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
976
+ end
977
+
978
+ let(:json_events) do
979
+ events = []
980
+ events << <<-END
981
+ {
982
+ "@timestamp": "2016-11-11T12:09:19.000Z",
983
+ "netflow": {
984
+ "netscalerHttpReqUserAgent": "Mozilla/5.0 (Commodore 64; kobo.com) Gecko/20100101 Firefox/75.0",
985
+ "destinationTransportPort": 443,
986
+ "netscalerHttpReqCookie": "beer=123456789abcdefghijklmnopqrstuvw; AnotherCookie=1234567890abcdefghijklmnopqr; Shameless.Plug=Thankyou.Rakuten.Kobo.Inc.For.Allowing.me.time.to.work.on.this.and.contribute.back.to.the.community; Padding=aaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbccccccccccccccddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffgggggggggggggggggggggggghhhhhhhhhhhhhhhhhiiiiiiiiiiiiiiiiiiiiiijjjjjjjjjjjjjjjjjjjjjjjjkkkkkkkkkkkkkkkkkklllllllllllllllmmmmmmmmmm; more=less; GJquote=There.is.no.spoon; GarrySays=Nice!!; LastPadding=aaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbcccccccccccccccccccdddddddddddeeeeeeee",
987
+ "flowEndMicroseconds": "503894-10-15T08:48:16.972Z",
988
+ "netscalerHttpReqUrl": "/aa/bb/ccccc/ddddddddddddddddddddddddd",
989
+ "sourceIPv4Address": "192.168.0.1",
990
+ "netscalerHttpReqMethod": "GET",
991
+ "netscalerHttpReqHost": "www.kobo.com",
992
+ "egressInterface": 2147483651,
993
+ "octetDeltaCount": 1541,
994
+ "netscalerAppNameAppId": 240189440,
995
+ "sourceTransportPort": 51053,
996
+ "flowId": 14460661,
997
+ "netscalerHttpReqAuthorization": "",
998
+ "netscalerHttpDomainName": "www.kobo.com",
999
+ "netscalerAaaUsername": "",
1000
+ "netscalerHttpContentType": "",
1001
+ "destinationIPv4Address": "10.0.0.1",
1002
+ "observationPointId": 167954698,
1003
+ "netscalerHttpReqVia": "1.1 akamai.net(ghost) (AkamaiGHost)",
1004
+ "netscalerConnectionId": 14460661,
1005
+ "tcpControlBits": 24,
1006
+ "flowStartMicroseconds": "503894-10-15T08:48:16.972Z",
1007
+ "ingressInterface": 8,
1008
+ "version": 10,
1009
+ "packetDeltaCount": 2,
1010
+ "netscalerUnknown330": 0,
1011
+ "netscalerConnectionChainID": "00e0ed1c9ca80300efb42558596b0800",
1012
+ "ipVersion": 4,
1013
+ "protocolIdentifier": 6,
1014
+ "netscalerHttpResForwLB": 0,
1015
+ "netscalerHttpReqReferer": "http://www.kobo.com/is-the-best-ebook-company-in-the-world",
1016
+ "exportingProcessId": 3,
1017
+ "netscalerAppUnitNameAppId": 239927296,
1018
+ "netscalerFlowFlags": 84025344,
1019
+ "netscalerTransactionId": 1068114985,
1020
+ "netscalerHttpResForwFB": 0,
1021
+ "netscalerConnectionChainHopCount": 1,
1022
+ "netscalerHttpReqXForwardedFor": "11.222.33.255"
1023
+ },
1024
+ "@version": "1"
1025
+ }
1026
+ END
1027
+ end
1028
+
1029
+ it "should decode raw data" do
1030
+ expect(decode.size).to eq(3)
1031
+ expect(decode[0].get("[netflow][version]")).to eq(10)
1032
+ expect(decode[0].get("[netflow][sourceIPv4Address]")).to eq('192.168.0.1')
1033
+ expect(decode[0].get("[netflow][destinationIPv4Address]")).to eq('10.0.0.1')
1034
+ expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('503894-10-15T08:48:16.970Z')
1035
+ expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
1036
+ expect(decode[1].get("[netflow][version]")).to eq(10)
1037
+ expect(decode[1].get("[netflow][flowId]")).to eq(14460662)
1038
+ expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
1039
+ expect(decode[1].get("[netflow][netscalerFlowFlags]")).to eq(1157636096)
1040
+ expect(decode[1].get("[netflow][netscalerRoundTripTime]")).to eq(83)
1041
+ expect(decode[2].get("[netflow][version]")).to eq(10)
1042
+ expect(decode[2].get("[netflow][netscalerAppUnitNameAppId]")).to eq(239927296)
1043
+ expect(decode[2].get("[netflow][netscalerHttpReqXForwardedFor]")).to eq('11.222.33.255')
1044
+ end
1045
+
1046
+ it "should decode variable length fields" do
1047
+ expect(decode[2].get("[netflow][netscalerHttpReqUrl]")).to eq('/aa/bb/ccccc/ddddddddddddddddddddddddd')
1048
+ expect(decode[2].get("[netflow][netscalerHttpReqHost]")).to eq('www.kobo.com')
1049
+ expect(decode[2].get("[netflow][netscalerHttpReqUserAgent]")).to eq('Mozilla/5.0 (Commodore 64; kobo.com) Gecko/20100101 Firefox/75.0')
1050
+ expect(decode[2].get("[netflow][netscalerHttpReqVia]")).to eq('1.1 akamai.net(ghost) (AkamaiGHost)')
1051
+ end
1052
+
1053
+ it "should decode fields with more than 255 chars" do
1054
+ expect(decode[2].get("[netflow][netscalerHttpReqCookie]")).to eq('beer=123456789abcdefghijklmnopqrstuvw; AnotherCookie=1234567890abcdefghijklmnopqr; Shameless.Plug=Thankyou.Rakuten.Kobo.Inc.For.Allowing.me.time.to.work.on.this.and.contribute.back.to.the.community; Padding=aaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbccccccccccccccddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffgggggggggggggggggggggggghhhhhhhhhhhhhhhhhiiiiiiiiiiiiiiiiiiiiiijjjjjjjjjjjjjjjjjjjjjjjjkkkkkkkkkkkkkkkkkklllllllllllllllmmmmmmmmmm; more=less; GJquote=There.is.no.spoon; GarrySays=Nice!!; LastPadding=aaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbcccccccccccccccccccdddddddddddeeeeeeee')
1055
+ end
1056
+
1057
+ it "should decode octetarray data" do
1058
+ expect(decode[0].get("[netflow][netscalerConnectionChainID]")).to eq('00e0ed1c9ca80300efb4255884850600')
1059
+ end
1060
+
1061
+ it "should serialize to json" do
1062
+ expect(JSON.parse(decode[2].to_json)).to eq(JSON.parse(json_events[0]))
1063
+ end
1064
+ end
1065
+ end
1066
+
1067
+ describe LogStash::Codecs::Netflow, 'missing templates, no template caching configured' do
1068
+ subject do
1069
+ LogStash::Codecs::Netflow.new.tap do |codec|
1070
+ expect{codec.register}.not_to raise_error
1071
+ end
1072
+ end
1073
+
1074
+ let(:logger) { double("logger") }
1075
+
1076
+ before :each do
1077
+ allow(LogStash::Codecs::Netflow).to receive(:logger).and_return(logger)
1078
+ allow(logger).to receive(:debug) {}
1079
+ allow(logger).to receive(:warn) {}
1080
+ end
1081
+
1082
+ let(:decode) do
1083
+ [].tap do |events|
1084
+ data.each { |packet| subject.decode(packet){|event| events << event}}
1085
+ end
1086
+ end
1087
+
1088
+ context "IPFIX Netscaler with variable length fields, missing templates" do
1089
+ let(:data) do
1090
+ data = []
1091
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
1092
+ end
1093
+
1094
+ it "can not / should not decode any data" do
1095
+ expect(decode.size).to eq(0)
1096
+ expect{decode[0].get("[netflow][version]")}.to raise_error(NoMethodError, /undefined method .get. for nil:NilClass/)
1097
+ expect{JSON.parse(decode[0].to_json)}.to raise_error(JSON::ParserError)
1098
+ end
1099
+
1100
+ it "should report missing templates" do
1101
+ expect(logger).to receive(:warn).with(/No matching template for flow id/)
1102
+ decode[0]
1103
+ end
1104
+ end
1105
+ end
1106
+
1107
+ # New subject with config, ordered testing since we need caching before data processing
1108
+ describe LogStash::Codecs::Netflow, 'configured with template caching', :order => :defined do
1109
+ context "IPFIX Netscaler with variable length fields" do
1110
+ subject do
1111
+ LogStash::Codecs::Netflow.new(cache_config).tap do |codec|
1112
+ expect{codec.register}.not_to raise_error
1113
+ end
1114
+ end
1115
+
1116
+ let(:tmp_dir) { ENV["TMP"] || ENV["TMPDIR"] || ENV["TEMP"] || "/tmp" }
1117
+
1118
+ let(:cache_config) do
1119
+ { "cache_save_path" => tmp_dir }
1120
+ end
1121
+
1122
+ let(:data) do
1123
+ # this ipfix raw data was produced by a Netscaler appliance and captured with wireshark
1124
+ # select packet bytes were then exported and sort of Pseudonymized to protect corp data
1125
+ data = []
1126
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
1127
+ end
1128
+
1129
+ let(:templates) do
1130
+ templates = []
1131
+ templates << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_tpl.dat"), :mode => "rb")
1132
+ end
1133
+
1134
+ let(:cache) do
1135
+ [].tap do |events|
1136
+ templates.each { |packet| subject.decode(packet){|event| events << event}}
1137
+ end
1138
+ end
1139
+
1140
+ let(:decode) do
1141
+ [].tap do |events|
1142
+ data.each { |packet| subject.decode(packet){|event| events << event}}
1143
+ end
1144
+ end
1145
+
1146
+ let(:cached_templates) do
1147
+ cached_templates = <<-END
1148
+ {
1149
+ "0|256": [
1150
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1151
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
1152
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1153
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
1154
+ ["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"]],
1155
+ "0|257": [
1156
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1157
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
1158
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1159
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","netscalerRoundTripTime"],["uint32","egressInterface"],
1160
+ ["uint32","ingressInterface"],["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],
1161
+ ["uint16","netscalerUnknown329"],["uint16","netscalerUnknown331"],["uint32","netscalerUnknown332"]],
1162
+ "0|258": [
1163
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1164
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
1165
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1166
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
1167
+ ["uint32","netscalerAppNameAppId"],["uint32","netscalerAppUnitNameAppId"],["uint64","netscalerHttpResForwFB"],["uint64","netscalerHttpResForwLB"],
1168
+ ["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"],
1169
+ ["VarString","netscalerAaaUsername"],["VarString","netscalerHttpReqUrl"],["VarString","netscalerHttpReqCookie"],["VarString","netscalerHttpReqReferer"],
1170
+ ["VarString","netscalerHttpReqMethod"],["VarString","netscalerHttpReqHost"],["VarString","netscalerHttpReqUserAgent"],["VarString","netscalerHttpContentType"],
1171
+ ["VarString","netscalerHttpReqAuthorization"],["VarString","netscalerHttpReqVia"],["VarString","netscalerHttpReqXForwardedFor"],["VarString","netscalerHttpDomainName"]],
1172
+ "0|259": [
1173
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1174
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip6_addr","sourceIPv6Address"],["ip6_addr","destinationIPv6Address"],
1175
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1176
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
1177
+ ["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"]],
1178
+ "0|260": [
1179
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1180
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip6_addr","sourceIPv6Address"],["ip6_addr","destinationIPv6Address"],
1181
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1182
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","netscalerRoundTripTime"],["uint32","egressInterface"],
1183
+ ["uint32","ingressInterface"],["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],
1184
+ ["uint16","netscalerUnknown329"],["uint16","netscalerUnknown331"],["uint32","netscalerUnknown332"]],
1185
+ "0|261": [
1186
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1187
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip6_addr","sourceIPv6Address"],["ip6_addr","destinationIPv6Address"],
1188
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1189
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
1190
+ ["uint32","netscalerAppNameAppId"],["uint32","netscalerAppUnitNameAppId"],["uint64","netscalerHttpResForwFB"],["uint64","netscalerHttpResForwLB"],
1191
+ ["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"],
1192
+ ["uint32","netscalerCacheRedirClientConnectionCoreID"],["uint32","netscalerCacheRedirClientConnectionTransactionID"],["VarString","netscalerAaaUsername"],
1193
+ ["VarString","netscalerHttpReqUrl"],["VarString","netscalerHttpReqCookie"],["VarString","netscalerHttpReqReferer"],["VarString","netscalerHttpReqMethod"],
1194
+ ["VarString","netscalerHttpReqHost"],["VarString","netscalerHttpReqUserAgent"],["VarString","netscalerHttpContentType"],["VarString","netscalerHttpReqAuthorization"],
1195
+ ["VarString","netscalerHttpReqVia"],["VarString","netscalerHttpReqXForwardedFor"],["VarString","netscalerHttpDomainName"]],
1196
+ "0|262": [
1197
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1198
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
1199
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1200
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
1201
+ ["uint16","netscalerHttpRspStatus"],["uint64","netscalerHttpRspLen"],["uint64","netscalerServerTTFB"],["uint64","netscalerServerTTLB"],
1202
+ ["uint32","netscalerAppNameAppId"],["uint32","netscalerMainPageId"],["uint32","netscalerMainPageCoreId"],["uint64","netscalerHttpReqRcvFB"],
1203
+ ["uint64","netscalerHttpReqForwFB"],["uint64","netscalerHttpResRcvFB"],["uint64","netscalerHttpReqRcvLB"],["uint64","netscalerHttpReqForwLB"],
1204
+ ["uint64","netscalerHttpResRcvLB"],["uint32","netscalerClientRTT"],["uint16","netscalerUnknown330"],["uint32","netscalerUnknown347"],["VarString","netscalerAaaUsername"],
1205
+ ["VarString","netscalerHttpContentType"],["VarString","netscalerHttpResLocation"],["VarString","netscalerHttpResSetCookie"],["VarString","netscalerHttpResSetCookie2"]]
1206
+ }
1207
+ END
1208
+ end
969
1209
 
1210
+ it "should cache templates" do
1211
+ expect(cache.size).to eq(0)
1212
+ expect(JSON.parse(File.read("#{tmp_dir}/ipfix_templates.cache"))).to eq(JSON.parse(cached_templates))
1213
+ end
1214
+
1215
+ it "should decode raw data based on cached templates" do
1216
+ expect(decode.size).to eq(3)
1217
+ expect(decode[0].get("[netflow][version]")).to eq(10)
1218
+ expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('503894-10-15T08:48:16.970Z')
1219
+ expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
1220
+ expect(decode[1].get("[netflow][version]")).to eq(10)
1221
+ expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
1222
+ expect(decode[1].get("[netflow][netscalerFlowFlags]")).to eq(1157636096)
1223
+ expect(decode[2].get("[netflow][version]")).to eq(10)
1224
+ expect(decode[2].get("[netflow][netscalerAppUnitNameAppId]")).to eq(239927296)
1225
+ expect(decode[2].get("[netflow][netscalerHttpReqXForwardedFor]")).to eq('11.222.33.255')
1226
+ FileUtils.rm_rf(tmp_dir)
1227
+ end
1228
+ end
1229
+ end
1230
+
1231
+ describe LogStash::Codecs::Netflow, 'configured with include_flowset_id for ipfix' do
1232
+ subject do
1233
+ LogStash::Codecs::Netflow.new(include_flowset_id_config).tap do |codec|
1234
+ expect{codec.register}.not_to raise_error
1235
+ end
1236
+ end
1237
+
1238
+ let(:include_flowset_id_config) do
1239
+ { "include_flowset_id" => true }
1240
+ end
1241
+
1242
+ let(:decode) do
1243
+ [].tap do |events|
1244
+ data.each { |packet| subject.decode(packet){|event| events << event}}
1245
+ end
1246
+ end
1247
+
1248
+ let(:data) do
1249
+ # this ipfix raw data was produced by a Netscaler appliance and captured with wireshark
1250
+ # select packet bytes were then exported and sort of Pseudonymized to protect corp data
1251
+ data = []
1252
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_tpl.dat"), :mode => "rb")
1253
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
1254
+ end
1255
+
1256
+ it "should decode raw data" do
1257
+ expect(decode.size).to eq(3)
1258
+ expect(decode[0].get("[netflow][version]")).to eq(10)
1259
+ expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('503894-10-15T08:48:16.970Z')
1260
+ expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
1261
+ expect(decode[1].get("[netflow][version]")).to eq(10)
1262
+ expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
1263
+ expect(decode[1].get("[netflow][netscalerFlowFlags]")).to eq(1157636096)
1264
+ expect(decode[2].get("[netflow][version]")).to eq(10)
1265
+ expect(decode[2].get("[netflow][netscalerAppUnitNameAppId]")).to eq(239927296)
1266
+ end
1267
+
1268
+ it "should include flowset_id" do
1269
+ expect(decode[0].get("[netflow][flowset_id]")).to eq(258)
1270
+ expect(decode[1].get("[netflow][flowset_id]")).to eq(257)
1271
+ expect(decode[2].get("[netflow][flowset_id]")).to eq(258)
1272
+ end
970
1273
  end
metadata CHANGED
@@ -1,81 +1,76 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-netflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.4
4
+ version: 3.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
  date: 2016-12-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ name: logstash-core-plugin-api
14
15
  requirement: !ruby/object:Gem::Requirement
15
16
  requirements:
16
- - - ">="
17
+ - - '>='
17
18
  - !ruby/object:Gem::Version
18
19
  version: '1.60'
19
- - - "<="
20
+ - - <=
20
21
  - !ruby/object:Gem::Version
21
22
  version: '2.99'
22
- name: logstash-core-plugin-api
23
- prerelease: false
24
23
  type: :runtime
24
+ prerelease: false
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
- - - ">="
27
+ - - '>='
28
28
  - !ruby/object:Gem::Version
29
29
  version: '1.60'
30
- - - "<="
30
+ - - <=
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2.99'
33
33
  - !ruby/object:Gem::Dependency
34
+ name: bindata
34
35
  requirement: !ruby/object:Gem::Requirement
35
36
  requirements:
36
- - - ">="
37
+ - - '>='
37
38
  - !ruby/object:Gem::Version
38
39
  version: 1.5.0
39
- name: bindata
40
- prerelease: false
41
40
  type: :runtime
41
+ prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - ">="
44
+ - - '>='
45
45
  - !ruby/object:Gem::Version
46
46
  version: 1.5.0
47
47
  - !ruby/object:Gem::Dependency
48
+ name: logstash-devutils
48
49
  requirement: !ruby/object:Gem::Requirement
49
50
  requirements:
50
- - - ">="
51
+ - - '>='
51
52
  - !ruby/object:Gem::Version
52
53
  version: 1.0.0
53
- name: logstash-devutils
54
- prerelease: false
55
54
  type: :development
55
+ prerelease: false
56
56
  version_requirements: !ruby/object:Gem::Requirement
57
57
  requirements:
58
- - - ">="
58
+ - - '>='
59
59
  - !ruby/object:Gem::Version
60
60
  version: 1.0.0
61
- description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
61
+ description: This gem is a Logstash plugin required to be installed on top of the
62
+ Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
63
+ gem is not a stand-alone program
62
64
  email: info@elastic.co
63
65
  executables: []
64
66
  extensions: []
65
67
  extra_rdoc_files: []
66
68
  files:
67
- - CHANGELOG.md
68
- - CONTRIBUTORS
69
- - Gemfile
70
- - LICENSE
71
- - NOTICE.TXT
72
- - README.md
73
- - lib/logstash/codecs/netflow.rb
74
69
  - lib/logstash/codecs/netflow/iana2yaml.rb
75
- - lib/logstash/codecs/netflow/ipfix.yaml
76
70
  - lib/logstash/codecs/netflow/netflow.yaml
71
+ - lib/logstash/codecs/netflow/ipfix.yaml
77
72
  - lib/logstash/codecs/netflow/util.rb
78
- - logstash-codec-netflow.gemspec
73
+ - lib/logstash/codecs/netflow.rb
79
74
  - spec/codecs/ipfix.dat
80
75
  - spec/codecs/ipfix_test_openbsd_pflow_data.dat
81
76
  - spec/codecs/ipfix_test_openbsd_pflow_tpl.dat
@@ -95,35 +90,44 @@ files:
95
90
  - spec/codecs/netflow9_test_nprobe_data.dat
96
91
  - spec/codecs/netflow9_test_nprobe_tpl.dat
97
92
  - spec/codecs/netflow9_test_softflowd_tpl_data.dat
93
+ - spec/codecs/netflow9_test_valid01.dat
94
+ - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
98
95
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
99
96
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
100
- - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
101
- - spec/codecs/netflow9_test_valid01.dat
97
+ - spec/codecs/ipfix_test_netscaler_data.dat
98
+ - spec/codecs/ipfix_test_netscaler_tpl.dat
102
99
  - spec/codecs/netflow_spec.rb
100
+ - logstash-codec-netflow.gemspec
101
+ - README.md
102
+ - CHANGELOG.md
103
+ - CONTRIBUTORS
104
+ - Gemfile
105
+ - LICENSE
106
+ - NOTICE.TXT
103
107
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
104
108
  licenses:
105
109
  - Apache License (2.0)
106
110
  metadata:
107
111
  logstash_plugin: 'true'
108
112
  logstash_group: codec
109
- post_install_message:
113
+ post_install_message:
110
114
  rdoc_options: []
111
115
  require_paths:
112
116
  - lib
113
117
  required_ruby_version: !ruby/object:Gem::Requirement
114
118
  requirements:
115
- - - ">="
119
+ - - '>='
116
120
  - !ruby/object:Gem::Version
117
121
  version: '0'
118
122
  required_rubygems_version: !ruby/object:Gem::Requirement
119
123
  requirements:
120
- - - ">="
124
+ - - '>='
121
125
  - !ruby/object:Gem::Version
122
126
  version: '0'
123
127
  requirements: []
124
- rubyforge_project:
125
- rubygems_version: 2.4.8
126
- signing_key:
128
+ rubyforge_project:
129
+ rubygems_version: 2.0.14
130
+ signing_key:
127
131
  specification_version: 4
128
132
  summary: The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows.
129
133
  test_files:
@@ -146,8 +150,10 @@ test_files:
146
150
  - spec/codecs/netflow9_test_nprobe_data.dat
147
151
  - spec/codecs/netflow9_test_nprobe_tpl.dat
148
152
  - spec/codecs/netflow9_test_softflowd_tpl_data.dat
153
+ - spec/codecs/netflow9_test_valid01.dat
154
+ - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
149
155
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
150
156
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
151
- - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
152
- - spec/codecs/netflow9_test_valid01.dat
157
+ - spec/codecs/ipfix_test_netscaler_data.dat
158
+ - spec/codecs/ipfix_test_netscaler_tpl.dat
153
159
  - spec/codecs/netflow_spec.rb