logstash-codec-netflow 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ MTc3NTE4OWRjM2RiYzc5MDhlNjViMTQ1MzZiNWFiMmIyODcwOGYwZg==
5
+ data.tar.gz: !binary |-
6
+ MjEzYTdhMDVkNzcwYjE1MWI5ZmY4MzFjZGFhYjQ1YmEzYWE1NzAyYQ==
7
+ SHA512:
8
+ metadata.gz: !binary |-
9
+ YTg3N2Q1NTVmMzcwOWY4YWYxM2MxMWI1YTIyYmY1NjQzOTQxMDgwYzZkYjAy
10
+ Nzk3ZjI5NTJjMzA4MjAyMWQ0YjFjNmU0NDViOTI1MmEyNWI4ZGMyNjJkYzdj
11
+ ZWU1MzhkMzAxZjM2Y2EyZmVlNzUwYmQzZjkwYjdmMDMzYmZkZTE=
12
+ data.tar.gz: !binary |-
13
+ OWJkODUxNDkxZDJhYTQ2YWY5Yzk3MDljMTBjNDkzOGJkZDZlZGFhY2E4NWE3
14
+ MGNkYTk0ZDMxODc3NGZmNTg2YTVjNjRkYzk3NTYyNzJhOWI4OTE2ZjM3MGRh
15
+ ZTk0MWIyNWI2Nzg5NjQ4NjQ5OGRkZDk0ZjIzNGJmNWMyN2Y0Y2Q=
@@ -0,0 +1,3 @@
1
+ *.gem
2
+ Gemfile.lock
3
+ .bundle
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'http://rubygems.org'
2
+ gem 'rake'
3
+ gem 'gem_publisher'
4
+ gem 'archive-tar-minitar'
data/LICENSE ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
@@ -0,0 +1,6 @@
1
+ @files=[]
2
+
3
+ task :default do
4
+ system("rake -T")
5
+ end
6
+
@@ -0,0 +1,263 @@
1
+ # encoding: utf-8
2
+ require "logstash/filters/base"
3
+ require "logstash/namespace"
4
+ require "logstash/timestamp"
5
+
6
+ # The "netflow" codec is for decoding Netflow v5/v9 flows.
7
+ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
8
+ config_name "netflow"
9
+ milestone 1
10
+
11
+ # Netflow v9 template cache TTL (minutes)
12
+ config :cache_ttl, :validate => :number, :default => 4000
13
+
14
+ # Specify into what field you want the Netflow data.
15
+ config :target, :validate => :string, :default => "netflow"
16
+
17
+ # Specify which Netflow versions you will accept.
18
+ config :versions, :validate => :array, :default => [5, 9]
19
+
20
+ # Override YAML file containing Netflow field definitions
21
+ #
22
+ # Each Netflow field is defined like so:
23
+ #
24
+ # ---
25
+ # id:
26
+ # - default length in bytes
27
+ # - :name
28
+ # id:
29
+ # - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
30
+ # - :name
31
+ # id:
32
+ # - :skip
33
+ #
34
+ # See <https://github.com/logstash/logstash/tree/v%VERSION%/lib/logstash/codecs/netflow/netflow.yaml> for the base set.
35
+ config :definitions, :validate => :path
36
+
37
+ public
38
+ def initialize(params={})
39
+ super(params)
40
+ @threadsafe = false
41
+ end
42
+
43
+ public
44
+ def register
45
+ require "logstash/codecs/netflow/util"
46
+ @templates = Vash.new()
47
+
48
+ # Path to default Netflow v9 field definitions
49
+ filename = ::File.expand_path('netflow/netflow.yaml', ::File.dirname(__FILE__))
50
+
51
+ begin
52
+ @fields = YAML.load_file(filename)
53
+ rescue Exception => e
54
+ raise "#{self.class.name}: Bad syntax in definitions file #{filename}"
55
+ end
56
+
57
+ # Allow the user to augment/override/rename the supported Netflow fields
58
+ if @definitions
59
+ raise "#{self.class.name}: definitions file #{@definitions} does not exists" unless File.exists?(@definitions)
60
+ begin
61
+ @fields.merge!(YAML.load_file(@definitions))
62
+ rescue Exception => e
63
+ raise "#{self.class.name}: Bad syntax in definitions file #{@definitions}"
64
+ end
65
+ end
66
+ end # def register
67
+
68
+ public
69
+ def decode(payload, &block)
70
+ header = Header.read(payload)
71
+
72
+ unless @versions.include?(header.version)
73
+ @logger.warn("Ignoring Netflow version v#{header.version}")
74
+ return
75
+ end
76
+
77
+ if header.version == 5
78
+ flowset = Netflow5PDU.read(payload)
79
+ elsif header.version == 9
80
+ flowset = Netflow9PDU.read(payload)
81
+ else
82
+ @logger.warn("Unsupported Netflow version v#{header.version}")
83
+ return
84
+ end
85
+
86
+ flowset.records.each do |record|
87
+ if flowset.version == 5
88
+ event = LogStash::Event.new
89
+
90
+ # FIXME Probably not doing this right WRT JRuby?
91
+ #
92
+ # The flowset header gives us the UTC epoch seconds along with
93
+ # residual nanoseconds so we can set @timestamp to that easily
94
+ event.timestamp = LogStash::Timestamp.at(flowset.unix_sec, flowset.unix_nsec / 1000)
95
+ event[@target] = {}
96
+
97
+ # Copy some of the pertinent fields in the header to the event
98
+ ['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records'].each do |f|
99
+ event[@target][f] = flowset[f]
100
+ end
101
+
102
+ # Create fields in the event from each field in the flow record
103
+ record.each_pair do |k,v|
104
+ case k.to_s
105
+ when /_switched$/
106
+ # The flow record sets the first and last times to the device
107
+ # uptime in milliseconds. Given the actual uptime is provided
108
+ # in the flowset header along with the epoch seconds we can
109
+ # convert these into absolute times
110
+ millis = flowset.uptime - v
111
+ seconds = flowset.unix_sec - (millis / 1000)
112
+ micros = (flowset.unix_nsec / 1000) - (millis % 1000)
113
+ if micros < 0
114
+ seconds--
115
+ micros += 1000000
116
+ end
117
+ # FIXME Again, probably doing this wrong WRT JRuby?
118
+ event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
119
+ else
120
+ event[@target][k.to_s] = v
121
+ end
122
+ end
123
+
124
+ yield event
125
+ elsif flowset.version == 9
126
+ case record.flowset_id
127
+ when 0
128
+ # Template flowset
129
+ record.flowset_data.templates.each do |template|
130
+ catch (:field) do
131
+ fields = []
132
+ template.fields.each do |field|
133
+ entry = netflow_field_for(field.field_type, field.field_length)
134
+ if ! entry
135
+ throw :field
136
+ end
137
+ fields += entry
138
+ end
139
+ # We get this far, we have a list of fields
140
+ #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
141
+ key = "#{flowset.source_id}|#{template.template_id}"
142
+ @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
143
+ # Purge any expired templates
144
+ @templates.cleanup!
145
+ end
146
+ end
147
+ when 1
148
+ # Options template flowset
149
+ record.flowset_data.templates.each do |template|
150
+ catch (:field) do
151
+ fields = []
152
+ template.option_fields.each do |field|
153
+ entry = netflow_field_for(field.field_type, field.field_length)
154
+ if ! entry
155
+ throw :field
156
+ end
157
+ fields += entry
158
+ end
159
+ # We get this far, we have a list of fields
160
+ #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
161
+ key = "#{flowset.source_id}|#{template.template_id}"
162
+ @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
163
+ # Purge any expired templates
164
+ @templates.cleanup!
165
+ end
166
+ end
167
+ when 256..65535
168
+ # Data flowset
169
+ #key = "#{flowset.source_id}|#{event["source"]}|#{record.flowset_id}"
170
+ key = "#{flowset.source_id}|#{record.flowset_id}"
171
+ template = @templates[key]
172
+
173
+ if ! template
174
+ #@logger.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
175
+ @logger.warn("No matching template for flow id #{record.flowset_id}")
176
+ next
177
+ end
178
+
179
+ length = record.flowset_length - 4
180
+
181
+ # Template shouldn't be longer than the record and there should
182
+ # be at most 3 padding bytes
183
+ if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
184
+ @logger.warn("Template length doesn't fit cleanly into flowset", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
185
+ next
186
+ end
187
+
188
+ array = BinData::Array.new(:type => template, :initial_length => length / template.num_bytes)
189
+
190
+ records = array.read(record.flowset_data)
191
+
192
+ records.each do |r|
193
+ event = LogStash::Event.new(
194
+ LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(flowset.unix_sec),
195
+ @target => {}
196
+ )
197
+
198
+ # Fewer fields in the v9 header
199
+ ['version', 'flow_seq_num'].each do |f|
200
+ event[@target][f] = flowset[f]
201
+ end
202
+
203
+ event[@target]['flowset_id'] = record.flowset_id
204
+
205
+ r.each_pair do |k,v|
206
+ case k.to_s
207
+ when /_switched$/
208
+ millis = flowset.uptime - v
209
+ seconds = flowset.unix_sec - (millis / 1000)
210
+ # v9 did away with the nanosecs field
211
+ micros = 1000000 - (millis % 1000)
212
+ event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
213
+ else
214
+ event[@target][k.to_s] = v
215
+ end
216
+ end
217
+
218
+ yield event
219
+ end
220
+ else
221
+ @logger.warn("Unsupported flowset id #{record.flowset_id}")
222
+ end
223
+ end
224
+ end
225
+ end # def filter
226
+
227
+ private
228
+ def uint_field(length, default)
229
+ # If length is 4, return :uint32, etc. and use default if length is 0
230
+ ("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
231
+ end # def uint_field
232
+
233
+ private
234
+ def netflow_field_for(type, length)
235
+ if @fields.include?(type)
236
+ field = @fields[type]
237
+ if field.is_a?(Array)
238
+
239
+ if field[0].is_a?(Integer)
240
+ field[0] = uint_field(length, field[0])
241
+ end
242
+
243
+ # Small bit of fixup for skip or string field types where the length
244
+ # is dynamic
245
+ case field[0]
246
+ when :skip
247
+ field += [nil, {:length => length}]
248
+ when :string
249
+ field += [{:length => length, :trim_padding => true}]
250
+ end
251
+
252
+ @logger.debug("Definition complete", :field => field)
253
+ [field]
254
+ else
255
+ @logger.warn("Definition should be an array", :field => field)
256
+ nil
257
+ end
258
+ else
259
+ @logger.warn("Unsupported field", :type => type, :length => length)
260
+ nil
261
+ end
262
+ end # def netflow_field_for
263
+ end # class LogStash::Filters::Netflow
@@ -0,0 +1,215 @@
1
+ ---
2
+ 1:
3
+ - 4
4
+ - :in_bytes
5
+ 2:
6
+ - 4
7
+ - :in_pkts
8
+ 3:
9
+ - 4
10
+ - :flows
11
+ 4:
12
+ - :uint8
13
+ - :protocol
14
+ 5:
15
+ - :uint8
16
+ - :src_tos
17
+ 6:
18
+ - :uint8
19
+ - :tcp_flags
20
+ 7:
21
+ - :uint16
22
+ - :l4_src_port
23
+ 8:
24
+ - :ip4_addr
25
+ - :ipv4_src_addr
26
+ 9:
27
+ - :uint8
28
+ - :src_mask
29
+ 10:
30
+ - 2
31
+ - :input_snmp
32
+ 11:
33
+ - :uint16
34
+ - :l4_dst_port
35
+ 12:
36
+ - :ip4_addr
37
+ - :ipv4_dst_addr
38
+ 13:
39
+ - :uint8
40
+ - :dst_mask
41
+ 14:
42
+ - 2
43
+ - :output_snmp
44
+ 15:
45
+ - :ip4_addr
46
+ - :ipv4_next_hop
47
+ 16:
48
+ - 2
49
+ - :src_as
50
+ 17:
51
+ - 2
52
+ - :dst_as
53
+ 18:
54
+ - :ip4_addr
55
+ - :bgp_ipv4_next_hop
56
+ 19:
57
+ - 4
58
+ - :mul_dst_pkts
59
+ 20:
60
+ - 4
61
+ - :mul_dst_bytes
62
+ 21:
63
+ - :uint32
64
+ - :last_switched
65
+ 22:
66
+ - :uint32
67
+ - :first_switched
68
+ 23:
69
+ - 4
70
+ - :out_bytes
71
+ 24:
72
+ - 4
73
+ - :out_pkts
74
+ 25:
75
+ - :uint16
76
+ - :min_pkt_length
77
+ 26:
78
+ - :uint16
79
+ - :max_pkt_length
80
+ 27:
81
+ - :ip6_addr
82
+ - :ipv6_src_addr
83
+ 28:
84
+ - :ip6_addr
85
+ - :ipv6_dst_addr
86
+ 29:
87
+ - :uint8
88
+ - :ipv6_src_mask
89
+ 30:
90
+ - :uint8
91
+ - :ipv6_dst_mask
92
+ 31:
93
+ - :uint24
94
+ - :ipv6_flow_label
95
+ 32:
96
+ - :uint16
97
+ - :icmp_type
98
+ 33:
99
+ - :uint8
100
+ - :mul_igmp_type
101
+ 34:
102
+ - :uint32
103
+ - :sampling_interval
104
+ 35:
105
+ - :uint8
106
+ - :sampling_algorithm
107
+ 36:
108
+ - :uint16
109
+ - :flow_active_timeout
110
+ 37:
111
+ - :uint16
112
+ - :flow_inactive_timeout
113
+ 38:
114
+ - :uint8
115
+ - :engine_type
116
+ 39:
117
+ - :uint8
118
+ - :engine_id
119
+ 40:
120
+ - 4
121
+ - :total_bytes_exp
122
+ 41:
123
+ - 4
124
+ - :total_pkts_exp
125
+ 42:
126
+ - 4
127
+ - :total_flows_exp
128
+ 43:
129
+ - :skip
130
+ 44:
131
+ - :ip4_addr
132
+ - :ipv4_src_prefix
133
+ 45:
134
+ - :ip4_addr
135
+ - :ipv4_dst_prefix
136
+ 46:
137
+ - :uint8
138
+ - :mpls_top_label_type
139
+ 47:
140
+ - :uint32
141
+ - :mpls_top_label_ip_addr
142
+ 48:
143
+ - 4
144
+ - :flow_sampler_id
145
+ 49:
146
+ - :uint8
147
+ - :flow_sampler_mode
148
+ 50:
149
+ - :uint32
150
+ - :flow_sampler_random_interval
151
+ 51:
152
+ - :skip
153
+ 52:
154
+ - :uint8
155
+ - :min_ttl
156
+ 53:
157
+ - :uint8
158
+ - :max_ttl
159
+ 54:
160
+ - :uint16
161
+ - :ipv4_ident
162
+ 55:
163
+ - :uint8
164
+ - :dst_tos
165
+ 56:
166
+ - :mac_addr
167
+ - :in_src_max
168
+ 57:
169
+ - :mac_addr
170
+ - :out_dst_max
171
+ 58:
172
+ - :uint16
173
+ - :src_vlan
174
+ 59:
175
+ - :uint16
176
+ - :dst_vlan
177
+ 60:
178
+ - :uint8
179
+ - :ip_protocol_version
180
+ 61:
181
+ - :uint8
182
+ - :direction
183
+ 62:
184
+ - :ip6_addr
185
+ - :ipv6_next_hop
186
+ 63:
187
+ - :ip6_addr
188
+ - :bgp_ipv6_next_hop
189
+ 64:
190
+ - :uint32
191
+ - :ipv6_option_headers
192
+ 64:
193
+ - :skip
194
+ 65:
195
+ - :skip
196
+ 66:
197
+ - :skip
198
+ 67:
199
+ - :skip
200
+ 68:
201
+ - :skip
202
+ 69:
203
+ - :skip
204
+ 80:
205
+ - :mac_addr
206
+ - :in_dst_mac
207
+ 81:
208
+ - :mac_addr
209
+ - :out_src_mac
210
+ 82:
211
+ - :string
212
+ - :if_name
213
+ 83:
214
+ - :string
215
+ - :if_desc
@@ -0,0 +1,212 @@
1
+ # encoding: utf-8
2
+ require "bindata"
3
+ require "ipaddr"
4
+
5
+ class IP4Addr < BinData::Primitive
6
+ endian :big
7
+ uint32 :storage
8
+
9
+ def set(val)
10
+ ip = IPAddr.new(val)
11
+ if ! ip.ipv4?
12
+ raise ArgumentError, "invalid IPv4 address '#{val}'"
13
+ end
14
+ self.storage = ip.to_i
15
+ end
16
+
17
+ def get
18
+ IPAddr.new_ntoh([self.storage].pack('N')).to_s
19
+ end
20
+ end
21
+
22
+ class IP6Addr < BinData::Primitive
23
+ endian :big
24
+ uint128 :storage
25
+
26
+ def set(val)
27
+ ip = IPAddr.new(val)
28
+ if ! ip.ipv6?
29
+ raise ArgumentError, "invalid IPv6 address `#{val}'"
30
+ end
31
+ self.storage = ip.to_i
32
+ end
33
+
34
+ def get
35
+ IPAddr.new_ntoh((0..7).map { |i|
36
+ (self.storage >> (112 - 16 * i)) & 0xffff
37
+ }.pack('n8')).to_s
38
+ end
39
+ end
40
+
41
+ class MacAddr < BinData::Primitive
42
+ array :bytes, :type => :uint8, :initial_length => 6
43
+
44
+ def set(val)
45
+ ints = val.split(/:/).collect { |int| int.to_i(16) }
46
+ self.bytes = ints
47
+ end
48
+
49
+ def get
50
+ self.bytes.collect { |byte| byte.to_s(16) }.join(":")
51
+ end
52
+ end
53
+
54
+ class Header < BinData::Record
55
+ endian :big
56
+ uint16 :version
57
+ end
58
+
59
+ class Netflow5PDU < BinData::Record
60
+ endian :big
61
+ uint16 :version
62
+ uint16 :flow_records
63
+ uint32 :uptime
64
+ uint32 :unix_sec
65
+ uint32 :unix_nsec
66
+ uint32 :flow_seq_num
67
+ uint8 :engine_type
68
+ uint8 :engine_id
69
+ bit2 :sampling_algorithm
70
+ bit14 :sampling_interval
71
+ array :records, :initial_length => :flow_records do
72
+ ip4_addr :ipv4_src_addr
73
+ ip4_addr :ipv4_dst_addr
74
+ ip4_addr :ipv4_next_hop
75
+ uint16 :input_snmp
76
+ uint16 :output_snmp
77
+ uint32 :in_pkts
78
+ uint32 :in_bytes
79
+ uint32 :first_switched
80
+ uint32 :last_switched
81
+ uint16 :l4_src_port
82
+ uint16 :l4_dst_port
83
+ skip :length => 1
84
+ uint8 :tcp_flags # Split up the TCP flags maybe?
85
+ uint8 :protocol
86
+ uint8 :src_tos
87
+ uint16 :src_as
88
+ uint16 :dst_as
89
+ uint8 :src_mask
90
+ uint8 :dst_mask
91
+ skip :length => 2
92
+ end
93
+ end
94
+
95
+ class TemplateFlowset < BinData::Record
96
+ endian :big
97
+ array :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
98
+ uint16 :template_id
99
+ uint16 :field_count
100
+ array :fields, :initial_length => :field_count do
101
+ uint16 :field_type
102
+ uint16 :field_length
103
+ end
104
+ end
105
+ end
106
+
107
+ class OptionFlowset < BinData::Record
108
+ endian :big
109
+ array :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
110
+ uint16 :template_id
111
+ uint16 :scope_length
112
+ uint16 :option_length
113
+ array :scope_fields, :initial_length => lambda { scope_length / 4 } do
114
+ uint16 :field_type
115
+ uint16 :field_length
116
+ end
117
+ array :option_fields, :initial_length => lambda { option_length / 4 } do
118
+ uint16 :field_type
119
+ uint16 :field_length
120
+ end
121
+ end
122
+ skip :length => lambda { templates.length.odd? ? 2 : 0 }
123
+ end
124
+
125
+ class Netflow9PDU < BinData::Record
126
+ endian :big
127
+ uint16 :version
128
+ uint16 :flow_records
129
+ uint32 :uptime
130
+ uint32 :unix_sec
131
+ uint32 :flow_seq_num
132
+ uint32 :source_id
133
+ array :records, :read_until => :eof do
134
+ uint16 :flowset_id
135
+ uint16 :flowset_length
136
+ choice :flowset_data, :selection => :flowset_id do
137
+ template_flowset 0
138
+ option_flowset 1
139
+ string :default, :read_length => lambda { flowset_length - 4 }
140
+ end
141
+ end
142
+ end
143
+
144
+ # https://gist.github.com/joshaven/184837
145
+ class Vash < Hash
146
+ def initialize(constructor = {})
147
+ @register ||= {}
148
+ if constructor.is_a?(Hash)
149
+ super()
150
+ merge(constructor)
151
+ else
152
+ super(constructor)
153
+ end
154
+ end
155
+
156
+ alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
157
+ alias_method :regular_reader, :[] unless method_defined?(:regular_reader)
158
+
159
+ def [](key)
160
+ sterilize(key)
161
+ clear(key) if expired?(key)
162
+ regular_reader(key)
163
+ end
164
+
165
+ def []=(key, *args)
166
+ if args.length == 2
167
+ value, ttl = args[1], args[0]
168
+ elsif args.length == 1
169
+ value, ttl = args[0], 60
170
+ else
171
+ raise ArgumentError, "Wrong number of arguments, expected 2 or 3, received: #{args.length+1}\n"+
172
+ "Example Usage: volatile_hash[:key]=value OR volatile_hash[:key, ttl]=value"
173
+ end
174
+ sterilize(key)
175
+ ttl(key, ttl)
176
+ regular_writer(key, value)
177
+ end
178
+
179
+ def merge(hsh)
180
+ hsh.map {|key,value| self[sterile(key)] = hsh[key]}
181
+ self
182
+ end
183
+
184
+ def cleanup!
185
+ now = Time.now.to_i
186
+ @register.map {|k,v| clear(k) if v < now}
187
+ end
188
+
189
+ def clear(key)
190
+ sterilize(key)
191
+ @register.delete key
192
+ self.delete key
193
+ end
194
+
195
+ private
196
+ def expired?(key)
197
+ Time.now.to_i > @register[key].to_i
198
+ end
199
+
200
+ def ttl(key, secs=60)
201
+ @register[key] = Time.now.to_i + secs.to_i
202
+ end
203
+
204
+ def sterile(key)
205
+ String === key ? key.chomp('!').chomp('=') : key.to_s.chomp('!').chomp('=').to_sym
206
+ end
207
+
208
+ def sterilize(key)
209
+ key = sterile(key)
210
+ end
211
+ end
212
+
@@ -0,0 +1,26 @@
1
+ Gem::Specification.new do |s|
2
+
3
+ s.name = 'logstash-codec-netflow'
4
+ s.version = '0.1.0'
5
+ s.licenses = ['Apache License (2.0)']
6
+ s.summary = "The netflow codec is for decoding Netflow v5/v9 flows."
7
+ s.description = "The netflow codec is for decoding Netflow v5/v9 flows."
8
+ s.authors = ["Elasticsearch"]
9
+ s.email = 'richard.pijnenburg@elasticsearch.com'
10
+ s.homepage = "http://logstash.net/"
11
+ s.require_paths = ["lib"]
12
+
13
+ # Files
14
+ s.files = `git ls-files`.split($\)
15
+
16
+ # Tests
17
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
+
19
+ # Special flag to let us know this is actually a logstash plugin
20
+ s.metadata = { "logstash_plugin" => "true", "group" => "codec" }
21
+
22
+ # Gem dependencies
23
+ s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
24
+ s.add_runtime_dependency 'bindata', ['>= 1.5.0']
25
+ end
26
+
@@ -0,0 +1,9 @@
1
+ require "gem_publisher"
2
+
3
+ desc "Publish gem to RubyGems.org"
4
+ task :publish_gem do |t|
5
+ gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
6
+ gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
7
+ puts "Published #{gem}" if gem
8
+ end
9
+
@@ -0,0 +1,169 @@
1
+ require "net/http"
2
+ require "uri"
3
+ require "digest/sha1"
4
+
5
+ def vendor(*args)
6
+ return File.join("vendor", *args)
7
+ end
8
+
9
+ directory "vendor/" => ["vendor"] do |task, args|
10
+ mkdir task.name
11
+ end
12
+
13
+ def fetch(url, sha1, output)
14
+
15
+ puts "Downloading #{url}"
16
+ actual_sha1 = download(url, output)
17
+
18
+ if actual_sha1 != sha1
19
+ fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
20
+ end
21
+ end # def fetch
22
+
23
+ def file_fetch(url, sha1)
24
+ filename = File.basename( URI(url).path )
25
+ output = "vendor/#{filename}"
26
+ task output => [ "vendor/" ] do
27
+ begin
28
+ actual_sha1 = file_sha1(output)
29
+ if actual_sha1 != sha1
30
+ fetch(url, sha1, output)
31
+ end
32
+ rescue Errno::ENOENT
33
+ fetch(url, sha1, output)
34
+ end
35
+ end.invoke
36
+
37
+ return output
38
+ end
39
+
40
+ def file_sha1(path)
41
+ digest = Digest::SHA1.new
42
+ fd = File.new(path, "r")
43
+ while true
44
+ begin
45
+ digest << fd.sysread(16384)
46
+ rescue EOFError
47
+ break
48
+ end
49
+ end
50
+ return digest.hexdigest
51
+ ensure
52
+ fd.close if fd
53
+ end
54
+
55
+ def download(url, output)
56
+ uri = URI(url)
57
+ digest = Digest::SHA1.new
58
+ tmp = "#{output}.tmp"
59
+ Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
60
+ request = Net::HTTP::Get.new(uri.path)
61
+ http.request(request) do |response|
62
+ fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
63
+ size = (response["content-length"].to_i || -1).to_f
64
+ count = 0
65
+ File.open(tmp, "w") do |fd|
66
+ response.read_body do |chunk|
67
+ fd.write(chunk)
68
+ digest << chunk
69
+ if size > 0 && $stdout.tty?
70
+ count += chunk.bytesize
71
+ $stdout.write(sprintf("\r%0.2f%%", count/size * 100))
72
+ end
73
+ end
74
+ end
75
+ $stdout.write("\r \r") if $stdout.tty?
76
+ end
77
+ end
78
+
79
+ File.rename(tmp, output)
80
+
81
+ return digest.hexdigest
82
+ rescue SocketError => e
83
+ puts "Failure while downloading #{url}: #{e}"
84
+ raise
85
+ ensure
86
+ File.unlink(tmp) if File.exist?(tmp)
87
+ end # def download
88
+
89
+ def untar(tarball, &block)
90
+ require "archive/tar/minitar"
91
+ tgz = Zlib::GzipReader.new(File.open(tarball))
92
+ # Pull out typesdb
93
+ tar = Archive::Tar::Minitar::Input.open(tgz)
94
+ tar.each do |entry|
95
+ path = block.call(entry)
96
+ next if path.nil?
97
+ parent = File.dirname(path)
98
+
99
+ mkdir_p parent unless File.directory?(parent)
100
+
101
+ # Skip this file if the output file is the same size
102
+ if entry.directory?
103
+ mkdir path unless File.directory?(path)
104
+ else
105
+ entry_mode = entry.instance_eval { @mode } & 0777
106
+ if File.exists?(path)
107
+ stat = File.stat(path)
108
+ # TODO(sissel): Submit a patch to archive-tar-minitar upstream to
109
+ # expose headers in the entry.
110
+ entry_size = entry.instance_eval { @size }
111
+ # If file sizes are same, skip writing.
112
+ next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
113
+ end
114
+ puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
115
+ File.open(path, "w") do |fd|
116
+ # eof? check lets us skip empty files. Necessary because the API provided by
117
+ # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
118
+ # IO object. Something about empty files in this EntryStream causes
119
+ # IO.copy_stream to throw "can't convert nil into String" on JRuby
120
+ # TODO(sissel): File a bug about this.
121
+ while !entry.eof?
122
+ chunk = entry.read(16384)
123
+ fd.write(chunk)
124
+ end
125
+ #IO.copy_stream(entry, fd)
126
+ end
127
+ File.chmod(entry_mode, path)
128
+ end
129
+ end
130
+ tar.close
131
+ File.unlink(tarball) if File.file?(tarball)
132
+ end # def untar
133
+
134
+ def ungz(file)
135
+
136
+ outpath = file.gsub('.gz', '')
137
+ tgz = Zlib::GzipReader.new(File.open(file))
138
+ begin
139
+ File.open(outpath, "w") do |out|
140
+ IO::copy_stream(tgz, out)
141
+ end
142
+ File.unlink(file)
143
+ rescue
144
+ File.unlink(outpath) if File.file?(outpath)
145
+ raise
146
+ end
147
+ tgz.close
148
+ end
149
+
150
+ desc "Process any vendor files required for this plugin"
151
+ task "vendor" do |task, args|
152
+
153
+ @files.each do |file|
154
+ download = file_fetch(file['url'], file['sha1'])
155
+ if download =~ /.tar.gz/
156
+ prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
157
+ untar(download) do |entry|
158
+ if !file['files'].nil?
159
+ next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
160
+ out = entry.full_name.split("/").last
161
+ end
162
+ File.join('vendor', out)
163
+ end
164
+ elsif download =~ /.gz/
165
+ ungz(download)
166
+ end
167
+ end
168
+
169
+ end
@@ -0,0 +1 @@
1
+ require 'spec_helper'
metadata ADDED
@@ -0,0 +1,93 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-codec-netflow
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Elasticsearch
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-11-05 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ! '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 1.4.0
20
+ - - <
21
+ - !ruby/object:Gem::Version
22
+ version: 2.0.0
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: 1.4.0
30
+ - - <
31
+ - !ruby/object:Gem::Version
32
+ version: 2.0.0
33
+ - !ruby/object:Gem::Dependency
34
+ name: bindata
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ! '>='
38
+ - !ruby/object:Gem::Version
39
+ version: !binary |-
40
+ MS41LjA=
41
+ type: :runtime
42
+ prerelease: false
43
+ version_requirements: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ! '>='
46
+ - !ruby/object:Gem::Version
47
+ version: !binary |-
48
+ MS41LjA=
49
+ description: The netflow codec is for decoding Netflow v5/v9 flows.
50
+ email: richard.pijnenburg@elasticsearch.com
51
+ executables: []
52
+ extensions: []
53
+ extra_rdoc_files: []
54
+ files:
55
+ - .gitignore
56
+ - Gemfile
57
+ - LICENSE
58
+ - Rakefile
59
+ - lib/logstash/codecs/netflow.rb
60
+ - lib/logstash/codecs/netflow/netflow.yaml
61
+ - lib/logstash/codecs/netflow/util.rb
62
+ - logstash-codec-netflow.gemspec
63
+ - rakelib/publish.rake
64
+ - rakelib/vendor.rake
65
+ - spec/codecs/netflow_spec.rb
66
+ homepage: http://logstash.net/
67
+ licenses:
68
+ - Apache License (2.0)
69
+ metadata:
70
+ logstash_plugin: 'true'
71
+ group: codec
72
+ post_install_message:
73
+ rdoc_options: []
74
+ require_paths:
75
+ - lib
76
+ required_ruby_version: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - ! '>='
79
+ - !ruby/object:Gem::Version
80
+ version: '0'
81
+ required_rubygems_version: !ruby/object:Gem::Requirement
82
+ requirements:
83
+ - - ! '>='
84
+ - !ruby/object:Gem::Version
85
+ version: '0'
86
+ requirements: []
87
+ rubyforge_project:
88
+ rubygems_version: 2.4.1
89
+ signing_key:
90
+ specification_version: 4
91
+ summary: The netflow codec is for decoding Netflow v5/v9 flows.
92
+ test_files:
93
+ - spec/codecs/netflow_spec.rb