logstash-codec-netflow 3.1.4 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -51,6 +51,36 @@ class MacAddr < BinData::Primitive
51
51
  end
52
52
  end
53
53
 
54
+ class VarSkip < BinData::Primitive
55
+ endian :big
56
+ uint8 :length_1
57
+ uint16 :length_2, :onlyif => lambda { length_1 == 255 }
58
+ skip :length => lambda { (length_1 == 255) ? length_2 : length_1 }
59
+
60
+ def get
61
+ ""
62
+ end
63
+ end
64
+
65
+ class VarString < BinData::Primitive
66
+ endian :big
67
+ uint8 :length_1
68
+ uint16 :length_2, :onlyif => lambda { length_1 == 255 }
69
+ string :data, :trim_padding => true, :length => lambda { (length_1 == 255) ? length_2 : length_1 }
70
+
71
+ def set(val)
72
+ self.data = val
73
+ end
74
+
75
+ def get
76
+ self.data
77
+ end
78
+
79
+ def snapshot
80
+ super.encode("ASCII-8BIT", "UTF-8", invalid: :replace, undef: :replace)
81
+ end
82
+ end
83
+
54
84
  class ACLIdASA < BinData::Primitive
55
85
  array :bytes, :type => :uint8, :initial_length => 12
56
86
 
@@ -78,6 +108,20 @@ class Forwarding_Status < BinData::Record
78
108
  bit6 :reason
79
109
  end
80
110
 
111
+ class OctetArray < BinData::Primitive
112
+ # arg_processor :octetarray
113
+ mandatory_parameter :initial_length
114
+ array :bytes, :type => :uint8, :initial_length => :initial_length
115
+
116
+ def set(val)
117
+ self.bytes = val.scan(/../).collect { |hex| hex.to_i(16)}
118
+ end
119
+
120
+ def get
121
+ self.bytes.collect { |byte| byte.value.to_s(16).rjust(2,'0') }.join
122
+ end
123
+ end
124
+
81
125
  class Header < BinData::Record
82
126
  endian :big
83
127
  uint16 :version
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-netflow'
4
- s.version = '3.1.4'
4
+ s.version = '3.2.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows."
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -966,5 +966,308 @@ describe LogStash::Codecs::Netflow do
966
966
 
967
967
  end
968
968
 
969
+ context "IPFIX Netscaler with variable length fields" do
970
+ let(:data) do
971
+ # this ipfix raw data was produced by a Netscaler appliance and captured with wireshark
972
+ # select packet bytes were then exported and sort of Pseudonymized to protect corp data
973
+ data = []
974
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_tpl.dat"), :mode => "rb")
975
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
976
+ end
977
+
978
+ let(:json_events) do
979
+ events = []
980
+ events << <<-END
981
+ {
982
+ "@timestamp": "2016-11-11T12:09:19.000Z",
983
+ "netflow": {
984
+ "netscalerHttpReqUserAgent": "Mozilla/5.0 (Commodore 64; kobo.com) Gecko/20100101 Firefox/75.0",
985
+ "destinationTransportPort": 443,
986
+ "netscalerHttpReqCookie": "beer=123456789abcdefghijklmnopqrstuvw; AnotherCookie=1234567890abcdefghijklmnopqr; Shameless.Plug=Thankyou.Rakuten.Kobo.Inc.For.Allowing.me.time.to.work.on.this.and.contribute.back.to.the.community; Padding=aaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbccccccccccccccddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffgggggggggggggggggggggggghhhhhhhhhhhhhhhhhiiiiiiiiiiiiiiiiiiiiiijjjjjjjjjjjjjjjjjjjjjjjjkkkkkkkkkkkkkkkkkklllllllllllllllmmmmmmmmmm; more=less; GJquote=There.is.no.spoon; GarrySays=Nice!!; LastPadding=aaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbcccccccccccccccccccdddddddddddeeeeeeee",
987
+ "flowEndMicroseconds": "503894-10-15T08:48:16.972Z",
988
+ "netscalerHttpReqUrl": "/aa/bb/ccccc/ddddddddddddddddddddddddd",
989
+ "sourceIPv4Address": "192.168.0.1",
990
+ "netscalerHttpReqMethod": "GET",
991
+ "netscalerHttpReqHost": "www.kobo.com",
992
+ "egressInterface": 2147483651,
993
+ "octetDeltaCount": 1541,
994
+ "netscalerAppNameAppId": 240189440,
995
+ "sourceTransportPort": 51053,
996
+ "flowId": 14460661,
997
+ "netscalerHttpReqAuthorization": "",
998
+ "netscalerHttpDomainName": "www.kobo.com",
999
+ "netscalerAaaUsername": "",
1000
+ "netscalerHttpContentType": "",
1001
+ "destinationIPv4Address": "10.0.0.1",
1002
+ "observationPointId": 167954698,
1003
+ "netscalerHttpReqVia": "1.1 akamai.net(ghost) (AkamaiGHost)",
1004
+ "netscalerConnectionId": 14460661,
1005
+ "tcpControlBits": 24,
1006
+ "flowStartMicroseconds": "503894-10-15T08:48:16.972Z",
1007
+ "ingressInterface": 8,
1008
+ "version": 10,
1009
+ "packetDeltaCount": 2,
1010
+ "netscalerUnknown330": 0,
1011
+ "netscalerConnectionChainID": "00e0ed1c9ca80300efb42558596b0800",
1012
+ "ipVersion": 4,
1013
+ "protocolIdentifier": 6,
1014
+ "netscalerHttpResForwLB": 0,
1015
+ "netscalerHttpReqReferer": "http://www.kobo.com/is-the-best-ebook-company-in-the-world",
1016
+ "exportingProcessId": 3,
1017
+ "netscalerAppUnitNameAppId": 239927296,
1018
+ "netscalerFlowFlags": 84025344,
1019
+ "netscalerTransactionId": 1068114985,
1020
+ "netscalerHttpResForwFB": 0,
1021
+ "netscalerConnectionChainHopCount": 1,
1022
+ "netscalerHttpReqXForwardedFor": "11.222.33.255"
1023
+ },
1024
+ "@version": "1"
1025
+ }
1026
+ END
1027
+ end
1028
+
1029
+ it "should decode raw data" do
1030
+ expect(decode.size).to eq(3)
1031
+ expect(decode[0].get("[netflow][version]")).to eq(10)
1032
+ expect(decode[0].get("[netflow][sourceIPv4Address]")).to eq('192.168.0.1')
1033
+ expect(decode[0].get("[netflow][destinationIPv4Address]")).to eq('10.0.0.1')
1034
+ expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('503894-10-15T08:48:16.970Z')
1035
+ expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
1036
+ expect(decode[1].get("[netflow][version]")).to eq(10)
1037
+ expect(decode[1].get("[netflow][flowId]")).to eq(14460662)
1038
+ expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
1039
+ expect(decode[1].get("[netflow][netscalerFlowFlags]")).to eq(1157636096)
1040
+ expect(decode[1].get("[netflow][netscalerRoundTripTime]")).to eq(83)
1041
+ expect(decode[2].get("[netflow][version]")).to eq(10)
1042
+ expect(decode[2].get("[netflow][netscalerAppUnitNameAppId]")).to eq(239927296)
1043
+ expect(decode[2].get("[netflow][netscalerHttpReqXForwardedFor]")).to eq('11.222.33.255')
1044
+ end
1045
+
1046
+ it "should decode variable length fields" do
1047
+ expect(decode[2].get("[netflow][netscalerHttpReqUrl]")).to eq('/aa/bb/ccccc/ddddddddddddddddddddddddd')
1048
+ expect(decode[2].get("[netflow][netscalerHttpReqHost]")).to eq('www.kobo.com')
1049
+ expect(decode[2].get("[netflow][netscalerHttpReqUserAgent]")).to eq('Mozilla/5.0 (Commodore 64; kobo.com) Gecko/20100101 Firefox/75.0')
1050
+ expect(decode[2].get("[netflow][netscalerHttpReqVia]")).to eq('1.1 akamai.net(ghost) (AkamaiGHost)')
1051
+ end
1052
+
1053
+ it "should decode fields with more than 255 chars" do
1054
+ expect(decode[2].get("[netflow][netscalerHttpReqCookie]")).to eq('beer=123456789abcdefghijklmnopqrstuvw; AnotherCookie=1234567890abcdefghijklmnopqr; Shameless.Plug=Thankyou.Rakuten.Kobo.Inc.For.Allowing.me.time.to.work.on.this.and.contribute.back.to.the.community; Padding=aaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbccccccccccccccddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffgggggggggggggggggggggggghhhhhhhhhhhhhhhhhiiiiiiiiiiiiiiiiiiiiiijjjjjjjjjjjjjjjjjjjjjjjjkkkkkkkkkkkkkkkkkklllllllllllllllmmmmmmmmmm; more=less; GJquote=There.is.no.spoon; GarrySays=Nice!!; LastPadding=aaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbcccccccccccccccccccdddddddddddeeeeeeee')
1055
+ end
1056
+
1057
+ it "should decode octetarray data" do
1058
+ expect(decode[0].get("[netflow][netscalerConnectionChainID]")).to eq('00e0ed1c9ca80300efb4255884850600')
1059
+ end
1060
+
1061
+ it "should serialize to json" do
1062
+ expect(JSON.parse(decode[2].to_json)).to eq(JSON.parse(json_events[0]))
1063
+ end
1064
+ end
1065
+ end
1066
+
1067
+ describe LogStash::Codecs::Netflow, 'missing templates, no template caching configured' do
1068
+ subject do
1069
+ LogStash::Codecs::Netflow.new.tap do |codec|
1070
+ expect{codec.register}.not_to raise_error
1071
+ end
1072
+ end
1073
+
1074
+ let(:logger) { double("logger") }
1075
+
1076
+ before :each do
1077
+ allow(LogStash::Codecs::Netflow).to receive(:logger).and_return(logger)
1078
+ allow(logger).to receive(:debug) {}
1079
+ allow(logger).to receive(:warn) {}
1080
+ end
1081
+
1082
+ let(:decode) do
1083
+ [].tap do |events|
1084
+ data.each { |packet| subject.decode(packet){|event| events << event}}
1085
+ end
1086
+ end
1087
+
1088
+ context "IPFIX Netscaler with variable length fields, missing templates" do
1089
+ let(:data) do
1090
+ data = []
1091
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
1092
+ end
1093
+
1094
+ it "can not / should not decode any data" do
1095
+ expect(decode.size).to eq(0)
1096
+ expect{decode[0].get("[netflow][version]")}.to raise_error(NoMethodError, /undefined method .get. for nil:NilClass/)
1097
+ expect{JSON.parse(decode[0].to_json)}.to raise_error(JSON::ParserError)
1098
+ end
1099
+
1100
+ it "should report missing templates" do
1101
+ expect(logger).to receive(:warn).with(/No matching template for flow id/)
1102
+ decode[0]
1103
+ end
1104
+ end
1105
+ end
1106
+
1107
+ # New subject with config, ordered testing since we need caching before data processing
1108
+ describe LogStash::Codecs::Netflow, 'configured with template caching', :order => :defined do
1109
+ context "IPFIX Netscaler with variable length fields" do
1110
+ subject do
1111
+ LogStash::Codecs::Netflow.new(cache_config).tap do |codec|
1112
+ expect{codec.register}.not_to raise_error
1113
+ end
1114
+ end
1115
+
1116
+ let(:tmp_dir) { ENV["TMP"] || ENV["TMPDIR"] || ENV["TEMP"] || "/tmp" }
1117
+
1118
+ let(:cache_config) do
1119
+ { "cache_save_path" => tmp_dir }
1120
+ end
1121
+
1122
+ let(:data) do
1123
+ # this ipfix raw data was produced by a Netscaler appliance and captured with wireshark
1124
+ # select packet bytes were then exported and sort of Pseudonymized to protect corp data
1125
+ data = []
1126
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
1127
+ end
1128
+
1129
+ let(:templates) do
1130
+ templates = []
1131
+ templates << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_tpl.dat"), :mode => "rb")
1132
+ end
1133
+
1134
+ let(:cache) do
1135
+ [].tap do |events|
1136
+ templates.each { |packet| subject.decode(packet){|event| events << event}}
1137
+ end
1138
+ end
1139
+
1140
+ let(:decode) do
1141
+ [].tap do |events|
1142
+ data.each { |packet| subject.decode(packet){|event| events << event}}
1143
+ end
1144
+ end
1145
+
1146
+ let(:cached_templates) do
1147
+ cached_templates = <<-END
1148
+ {
1149
+ "0|256": [
1150
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1151
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
1152
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1153
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
1154
+ ["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"]],
1155
+ "0|257": [
1156
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1157
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
1158
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1159
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","netscalerRoundTripTime"],["uint32","egressInterface"],
1160
+ ["uint32","ingressInterface"],["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],
1161
+ ["uint16","netscalerUnknown329"],["uint16","netscalerUnknown331"],["uint32","netscalerUnknown332"]],
1162
+ "0|258": [
1163
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1164
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
1165
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1166
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
1167
+ ["uint32","netscalerAppNameAppId"],["uint32","netscalerAppUnitNameAppId"],["uint64","netscalerHttpResForwFB"],["uint64","netscalerHttpResForwLB"],
1168
+ ["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"],
1169
+ ["VarString","netscalerAaaUsername"],["VarString","netscalerHttpReqUrl"],["VarString","netscalerHttpReqCookie"],["VarString","netscalerHttpReqReferer"],
1170
+ ["VarString","netscalerHttpReqMethod"],["VarString","netscalerHttpReqHost"],["VarString","netscalerHttpReqUserAgent"],["VarString","netscalerHttpContentType"],
1171
+ ["VarString","netscalerHttpReqAuthorization"],["VarString","netscalerHttpReqVia"],["VarString","netscalerHttpReqXForwardedFor"],["VarString","netscalerHttpDomainName"]],
1172
+ "0|259": [
1173
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1174
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip6_addr","sourceIPv6Address"],["ip6_addr","destinationIPv6Address"],
1175
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1176
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
1177
+ ["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"]],
1178
+ "0|260": [
1179
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1180
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip6_addr","sourceIPv6Address"],["ip6_addr","destinationIPv6Address"],
1181
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1182
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","netscalerRoundTripTime"],["uint32","egressInterface"],
1183
+ ["uint32","ingressInterface"],["uint32","netscalerAppNameAppId"],["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],
1184
+ ["uint16","netscalerUnknown329"],["uint16","netscalerUnknown331"],["uint32","netscalerUnknown332"]],
1185
+ "0|261": [
1186
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1187
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip6_addr","sourceIPv6Address"],["ip6_addr","destinationIPv6Address"],
1188
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1189
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
1190
+ ["uint32","netscalerAppNameAppId"],["uint32","netscalerAppUnitNameAppId"],["uint64","netscalerHttpResForwFB"],["uint64","netscalerHttpResForwLB"],
1191
+ ["OctetArray","netscalerConnectionChainID",{"initial_length":16}],["uint8","netscalerConnectionChainHopCount"],["uint16","netscalerUnknown330"],
1192
+ ["uint32","netscalerCacheRedirClientConnectionCoreID"],["uint32","netscalerCacheRedirClientConnectionTransactionID"],["VarString","netscalerAaaUsername"],
1193
+ ["VarString","netscalerHttpReqUrl"],["VarString","netscalerHttpReqCookie"],["VarString","netscalerHttpReqReferer"],["VarString","netscalerHttpReqMethod"],
1194
+ ["VarString","netscalerHttpReqHost"],["VarString","netscalerHttpReqUserAgent"],["VarString","netscalerHttpContentType"],["VarString","netscalerHttpReqAuthorization"],
1195
+ ["VarString","netscalerHttpReqVia"],["VarString","netscalerHttpReqXForwardedFor"],["VarString","netscalerHttpDomainName"]],
1196
+ "0|262": [
1197
+ ["uint32","observationPointId"],["uint32","exportingProcessId"],["uint64","flowId"],["uint32","netscalerTransactionId"],["uint32","netscalerConnectionId"],
1198
+ ["uint8","ipVersion"],["uint8","protocolIdentifier"],["skip",null,{"length":2}],["ip4_addr","sourceIPv4Address"],["ip4_addr","destinationIPv4Address"],
1199
+ ["uint16","sourceTransportPort"],["uint16","destinationTransportPort"],["uint64","packetDeltaCount"],["uint64","octetDeltaCount"],["uint8","tcpControlBits"],
1200
+ ["uint64","netscalerFlowFlags"],["uint64","flowStartMicroseconds"],["uint64","flowEndMicroseconds"],["uint32","ingressInterface"],["uint32","egressInterface"],
1201
+ ["uint16","netscalerHttpRspStatus"],["uint64","netscalerHttpRspLen"],["uint64","netscalerServerTTFB"],["uint64","netscalerServerTTLB"],
1202
+ ["uint32","netscalerAppNameAppId"],["uint32","netscalerMainPageId"],["uint32","netscalerMainPageCoreId"],["uint64","netscalerHttpReqRcvFB"],
1203
+ ["uint64","netscalerHttpReqForwFB"],["uint64","netscalerHttpResRcvFB"],["uint64","netscalerHttpReqRcvLB"],["uint64","netscalerHttpReqForwLB"],
1204
+ ["uint64","netscalerHttpResRcvLB"],["uint32","netscalerClientRTT"],["uint16","netscalerUnknown330"],["uint32","netscalerUnknown347"],["VarString","netscalerAaaUsername"],
1205
+ ["VarString","netscalerHttpContentType"],["VarString","netscalerHttpResLocation"],["VarString","netscalerHttpResSetCookie"],["VarString","netscalerHttpResSetCookie2"]]
1206
+ }
1207
+ END
1208
+ end
969
1209
 
1210
+ it "should cache templates" do
1211
+ expect(cache.size).to eq(0)
1212
+ expect(JSON.parse(File.read("#{tmp_dir}/ipfix_templates.cache"))).to eq(JSON.parse(cached_templates))
1213
+ end
1214
+
1215
+ it "should decode raw data based on cached templates" do
1216
+ expect(decode.size).to eq(3)
1217
+ expect(decode[0].get("[netflow][version]")).to eq(10)
1218
+ expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('503894-10-15T08:48:16.970Z')
1219
+ expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
1220
+ expect(decode[1].get("[netflow][version]")).to eq(10)
1221
+ expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
1222
+ expect(decode[1].get("[netflow][netscalerFlowFlags]")).to eq(1157636096)
1223
+ expect(decode[2].get("[netflow][version]")).to eq(10)
1224
+ expect(decode[2].get("[netflow][netscalerAppUnitNameAppId]")).to eq(239927296)
1225
+ expect(decode[2].get("[netflow][netscalerHttpReqXForwardedFor]")).to eq('11.222.33.255')
1226
+ FileUtils.rm_rf(tmp_dir)
1227
+ end
1228
+ end
1229
+ end
1230
+
1231
+ describe LogStash::Codecs::Netflow, 'configured with include_flowset_id for ipfix' do
1232
+ subject do
1233
+ LogStash::Codecs::Netflow.new(include_flowset_id_config).tap do |codec|
1234
+ expect{codec.register}.not_to raise_error
1235
+ end
1236
+ end
1237
+
1238
+ let(:include_flowset_id_config) do
1239
+ { "include_flowset_id" => true }
1240
+ end
1241
+
1242
+ let(:decode) do
1243
+ [].tap do |events|
1244
+ data.each { |packet| subject.decode(packet){|event| events << event}}
1245
+ end
1246
+ end
1247
+
1248
+ let(:data) do
1249
+ # this ipfix raw data was produced by a Netscaler appliance and captured with wireshark
1250
+ # select packet bytes were then exported and sort of Pseudonymized to protect corp data
1251
+ data = []
1252
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_tpl.dat"), :mode => "rb")
1253
+ data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
1254
+ end
1255
+
1256
+ it "should decode raw data" do
1257
+ expect(decode.size).to eq(3)
1258
+ expect(decode[0].get("[netflow][version]")).to eq(10)
1259
+ expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('503894-10-15T08:48:16.970Z')
1260
+ expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
1261
+ expect(decode[1].get("[netflow][version]")).to eq(10)
1262
+ expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
1263
+ expect(decode[1].get("[netflow][netscalerFlowFlags]")).to eq(1157636096)
1264
+ expect(decode[2].get("[netflow][version]")).to eq(10)
1265
+ expect(decode[2].get("[netflow][netscalerAppUnitNameAppId]")).to eq(239927296)
1266
+ end
1267
+
1268
+ it "should include flowset_id" do
1269
+ expect(decode[0].get("[netflow][flowset_id]")).to eq(258)
1270
+ expect(decode[1].get("[netflow][flowset_id]")).to eq(257)
1271
+ expect(decode[2].get("[netflow][flowset_id]")).to eq(258)
1272
+ end
970
1273
  end
metadata CHANGED
@@ -1,81 +1,76 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-netflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.4
4
+ version: 3.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
  date: 2016-12-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ name: logstash-core-plugin-api
14
15
  requirement: !ruby/object:Gem::Requirement
15
16
  requirements:
16
- - - ">="
17
+ - - '>='
17
18
  - !ruby/object:Gem::Version
18
19
  version: '1.60'
19
- - - "<="
20
+ - - <=
20
21
  - !ruby/object:Gem::Version
21
22
  version: '2.99'
22
- name: logstash-core-plugin-api
23
- prerelease: false
24
23
  type: :runtime
24
+ prerelease: false
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
- - - ">="
27
+ - - '>='
28
28
  - !ruby/object:Gem::Version
29
29
  version: '1.60'
30
- - - "<="
30
+ - - <=
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2.99'
33
33
  - !ruby/object:Gem::Dependency
34
+ name: bindata
34
35
  requirement: !ruby/object:Gem::Requirement
35
36
  requirements:
36
- - - ">="
37
+ - - '>='
37
38
  - !ruby/object:Gem::Version
38
39
  version: 1.5.0
39
- name: bindata
40
- prerelease: false
41
40
  type: :runtime
41
+ prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - ">="
44
+ - - '>='
45
45
  - !ruby/object:Gem::Version
46
46
  version: 1.5.0
47
47
  - !ruby/object:Gem::Dependency
48
+ name: logstash-devutils
48
49
  requirement: !ruby/object:Gem::Requirement
49
50
  requirements:
50
- - - ">="
51
+ - - '>='
51
52
  - !ruby/object:Gem::Version
52
53
  version: 1.0.0
53
- name: logstash-devutils
54
- prerelease: false
55
54
  type: :development
55
+ prerelease: false
56
56
  version_requirements: !ruby/object:Gem::Requirement
57
57
  requirements:
58
- - - ">="
58
+ - - '>='
59
59
  - !ruby/object:Gem::Version
60
60
  version: 1.0.0
61
- description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
61
+ description: This gem is a Logstash plugin required to be installed on top of the
62
+ Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
63
+ gem is not a stand-alone program
62
64
  email: info@elastic.co
63
65
  executables: []
64
66
  extensions: []
65
67
  extra_rdoc_files: []
66
68
  files:
67
- - CHANGELOG.md
68
- - CONTRIBUTORS
69
- - Gemfile
70
- - LICENSE
71
- - NOTICE.TXT
72
- - README.md
73
- - lib/logstash/codecs/netflow.rb
74
69
  - lib/logstash/codecs/netflow/iana2yaml.rb
75
- - lib/logstash/codecs/netflow/ipfix.yaml
76
70
  - lib/logstash/codecs/netflow/netflow.yaml
71
+ - lib/logstash/codecs/netflow/ipfix.yaml
77
72
  - lib/logstash/codecs/netflow/util.rb
78
- - logstash-codec-netflow.gemspec
73
+ - lib/logstash/codecs/netflow.rb
79
74
  - spec/codecs/ipfix.dat
80
75
  - spec/codecs/ipfix_test_openbsd_pflow_data.dat
81
76
  - spec/codecs/ipfix_test_openbsd_pflow_tpl.dat
@@ -95,35 +90,44 @@ files:
95
90
  - spec/codecs/netflow9_test_nprobe_data.dat
96
91
  - spec/codecs/netflow9_test_nprobe_tpl.dat
97
92
  - spec/codecs/netflow9_test_softflowd_tpl_data.dat
93
+ - spec/codecs/netflow9_test_valid01.dat
94
+ - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
98
95
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
99
96
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
100
- - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
101
- - spec/codecs/netflow9_test_valid01.dat
97
+ - spec/codecs/ipfix_test_netscaler_data.dat
98
+ - spec/codecs/ipfix_test_netscaler_tpl.dat
102
99
  - spec/codecs/netflow_spec.rb
100
+ - logstash-codec-netflow.gemspec
101
+ - README.md
102
+ - CHANGELOG.md
103
+ - CONTRIBUTORS
104
+ - Gemfile
105
+ - LICENSE
106
+ - NOTICE.TXT
103
107
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
104
108
  licenses:
105
109
  - Apache License (2.0)
106
110
  metadata:
107
111
  logstash_plugin: 'true'
108
112
  logstash_group: codec
109
- post_install_message:
113
+ post_install_message:
110
114
  rdoc_options: []
111
115
  require_paths:
112
116
  - lib
113
117
  required_ruby_version: !ruby/object:Gem::Requirement
114
118
  requirements:
115
- - - ">="
119
+ - - '>='
116
120
  - !ruby/object:Gem::Version
117
121
  version: '0'
118
122
  required_rubygems_version: !ruby/object:Gem::Requirement
119
123
  requirements:
120
- - - ">="
124
+ - - '>='
121
125
  - !ruby/object:Gem::Version
122
126
  version: '0'
123
127
  requirements: []
124
- rubyforge_project:
125
- rubygems_version: 2.4.8
126
- signing_key:
128
+ rubyforge_project:
129
+ rubygems_version: 2.0.14
130
+ signing_key:
127
131
  specification_version: 4
128
132
  summary: The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows.
129
133
  test_files:
@@ -146,8 +150,10 @@ test_files:
146
150
  - spec/codecs/netflow9_test_nprobe_data.dat
147
151
  - spec/codecs/netflow9_test_nprobe_tpl.dat
148
152
  - spec/codecs/netflow9_test_softflowd_tpl_data.dat
153
+ - spec/codecs/netflow9_test_valid01.dat
154
+ - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
149
155
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
150
156
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
151
- - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
152
- - spec/codecs/netflow9_test_valid01.dat
157
+ - spec/codecs/ipfix_test_netscaler_data.dat
158
+ - spec/codecs/ipfix_test_netscaler_tpl.dat
153
159
  - spec/codecs/netflow_spec.rb