vanken 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +84 -0
- data/docs/filters.md +25 -0
- data/docs/performance.md +113 -0
- data/docs/releases.md +15 -0
- data/docs/upstream.md +32 -0
- data/exe/vanken +5 -0
- data/exe/vanken-capture +6 -0
- data/lib/vanken/app/capture_controller.rb +197 -0
- data/lib/vanken/app/document.rb +305 -0
- data/lib/vanken/app/document_jobs.rb +199 -0
- data/lib/vanken/capture/analyzer.rb +44 -0
- data/lib/vanken/capture/analyzer_process.rb +91 -0
- data/lib/vanken/capture/analyzer_wire.rb +42 -0
- data/lib/vanken/capture/analyzer_worker.rb +153 -0
- data/lib/vanken/capture/control_protocol.rb +32 -0
- data/lib/vanken/capture/filter_worker.rb +180 -0
- data/lib/vanken/capture/helper_main.rb +137 -0
- data/lib/vanken/capture/helper_options.rb +70 -0
- data/lib/vanken/capture/launcher.rb +181 -0
- data/lib/vanken/capture/privileges.rb +24 -0
- data/lib/vanken/capture/receiver.rb +39 -0
- data/lib/vanken/cli.rb +68 -0
- data/lib/vanken/config/log.rb +19 -0
- data/lib/vanken/config/preferences.rb +126 -0
- data/lib/vanken/core/display_filter/compiler.rb +108 -0
- data/lib/vanken/core/display_filter/errors.rb +19 -0
- data/lib/vanken/core/display_filter/field_resolver.rb +86 -0
- data/lib/vanken/core/display_filter/lexer.rb +124 -0
- data/lib/vanken/core/display_filter/parser.rb +157 -0
- data/lib/vanken/core/display_filter/types.rb +176 -0
- data/lib/vanken/core/frame.rb +33 -0
- data/lib/vanken/core/frame_store.rb +179 -0
- data/lib/vanken/core/frame_view.rb +68 -0
- data/lib/vanken/core/stores.rb +192 -0
- data/lib/vanken/errors.rb +9 -0
- data/lib/vanken/gateway/capture_filter.rb +30 -0
- data/lib/vanken/gateway/detail_builder.rb +63 -0
- data/lib/vanken/gateway/dissector.rb +176 -0
- data/lib/vanken/gateway/file_reader.rb +153 -0
- data/lib/vanken/gateway/file_writer.rb +64 -0
- data/lib/vanken/gateway/interfaces.rb +30 -0
- data/lib/vanken/gateway/live_capture.rb +79 -0
- data/lib/vanken/gateway/packet_snapshot.rb +89 -0
- data/lib/vanken/ui/actions.rb +50 -0
- data/lib/vanken/ui/application.rb +205 -0
- data/lib/vanken/ui/dialogs.rb +89 -0
- data/lib/vanken/ui/file_operations.rb +113 -0
- data/lib/vanken/ui/filter_operations.rb +120 -0
- data/lib/vanken/ui/main_view.rb +113 -0
- data/lib/vanken/ui/packet_source.rb +161 -0
- data/lib/vanken/ui/selection.rb +91 -0
- data/lib/vanken/version.rb +5 -0
- data/lib/vanken.rb +17 -0
- data/packaging/README.md +33 -0
- data/sig/generated/vanken/app/capture_controller.rbs +58 -0
- data/sig/generated/vanken/app/document.rbs +117 -0
- data/sig/generated/vanken/app/document_jobs.rbs +27 -0
- data/sig/generated/vanken/capture/analyzer.rbs +11 -0
- data/sig/generated/vanken/capture/analyzer_process.rbs +17 -0
- data/sig/generated/vanken/capture/analyzer_wire.rbs +19 -0
- data/sig/generated/vanken/capture/analyzer_worker.rbs +45 -0
- data/sig/generated/vanken/capture/control_protocol.rbs +17 -0
- data/sig/generated/vanken/capture/filter_worker.rbs +51 -0
- data/sig/generated/vanken/capture/helper_main.rbs +23 -0
- data/sig/generated/vanken/capture/helper_options.rbs +27 -0
- data/sig/generated/vanken/capture/launcher.rbs +60 -0
- data/sig/generated/vanken/capture/privileges.rbs +12 -0
- data/sig/generated/vanken/capture/receiver.rbs +15 -0
- data/sig/generated/vanken/cli.rbs +9 -0
- data/sig/generated/vanken/config/log.rbs +9 -0
- data/sig/generated/vanken/config/preferences.rbs +49 -0
- data/sig/generated/vanken/core/display_filter/compiler.rbs +45 -0
- data/sig/generated/vanken/core/display_filter/errors.rbs +16 -0
- data/sig/generated/vanken/core/display_filter/field_resolver.rbs +48 -0
- data/sig/generated/vanken/core/display_filter/lexer.rbs +49 -0
- data/sig/generated/vanken/core/display_filter/parser.rbs +95 -0
- data/sig/generated/vanken/core/display_filter/types.rbs +45 -0
- data/sig/generated/vanken/core/frame.rbs +63 -0
- data/sig/generated/vanken/core/frame_store.rbs +65 -0
- data/sig/generated/vanken/core/frame_view.rbs +36 -0
- data/sig/generated/vanken/core/stores.rbs +110 -0
- data/sig/generated/vanken/errors.rbs +18 -0
- data/sig/generated/vanken/gateway/capture_filter.rbs +17 -0
- data/sig/generated/vanken/gateway/detail_builder.rbs +21 -0
- data/sig/generated/vanken/gateway/dissector.rbs +113 -0
- data/sig/generated/vanken/gateway/file_reader.rbs +53 -0
- data/sig/generated/vanken/gateway/file_writer.rbs +27 -0
- data/sig/generated/vanken/gateway/interfaces.rbs +16 -0
- data/sig/generated/vanken/gateway/live_capture.rbs +48 -0
- data/sig/generated/vanken/gateway/packet_snapshot.rbs +48 -0
- data/sig/generated/vanken/ui/actions.rbs +9 -0
- data/sig/generated/vanken/ui/application.rbs +85 -0
- data/sig/generated/vanken/ui/dialogs.rbs +21 -0
- data/sig/generated/vanken/ui/file_operations.rbs +25 -0
- data/sig/generated/vanken/ui/filter_operations.rbs +33 -0
- data/sig/generated/vanken/ui/main_view.rbs +21 -0
- data/sig/generated/vanken/ui/packet_source.rbs +40 -0
- data/sig/generated/vanken/ui/selection.rbs +23 -0
- data/sig/generated/vanken/version.rbs +5 -0
- data/sig/vanken.rbs +25 -0
- metadata +192 -0
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../vanken"
|
|
4
|
+
require_relative "analyzer_wire"
|
|
5
|
+
|
|
6
|
+
module Vanken
|
|
7
|
+
module Capture
|
|
8
|
+
class AnalyzerWorker
|
|
9
|
+
class Reader
|
|
10
|
+
attr_accessor :limit, :interfaces
|
|
11
|
+
def initialize(directory)
|
|
12
|
+
@records = File.open(File.join(directory, "frames.idx"), "rb")
|
|
13
|
+
@data = File.open(File.join(directory, "frames.bin"), "rb")
|
|
14
|
+
@limit, @interfaces = 0, []
|
|
15
|
+
end
|
|
16
|
+
def metadata(number)
|
|
17
|
+
raise Vanken::Error, "frame outside published range" unless number.between?(1, @limit)
|
|
18
|
+
record = read_at(@records, Core::FrameStore::RECORD_SIZE, (number - 1) * Core::FrameStore::RECORD_SIZE)
|
|
19
|
+
raise Vanken::Error, "truncated analysis index" unless record.bytesize == Core::FrameStore::RECORD_SIZE
|
|
20
|
+
offset, caplen, original, time, link, interface, direction = record.unpack(Core::FrameStore::RECORD)
|
|
21
|
+
raise Vanken::Error, "invalid analysis frame" unless caplen <= 16 << 20 && original >= caplen && direction <= 2
|
|
22
|
+
{offset: offset, caplen: caplen, original_length: original, timestamp_ns: time, linktype: link,
|
|
23
|
+
interface: interface == 0xffff ? nil : @interfaces.fetch(interface), direction: [nil, :in, :out].fetch(direction), number: number}
|
|
24
|
+
end
|
|
25
|
+
def read(number)
|
|
26
|
+
meta = metadata(number)
|
|
27
|
+
bytes = meta[:caplen].zero? ? "".b : read_at(@data, meta[:caplen], meta[:offset])
|
|
28
|
+
raise Vanken::Error, "truncated analysis frame" unless bytes.bytesize == meta[:caplen]
|
|
29
|
+
Core::Frame.new(bytes: bytes, timestamp_ns: meta[:timestamp_ns], original_length: meta[:original_length],
|
|
30
|
+
linktype: meta[:linktype], interface: meta[:interface], direction: meta[:direction], number: number)
|
|
31
|
+
end
|
|
32
|
+
def close
|
|
33
|
+
@records.close
|
|
34
|
+
@data.close
|
|
35
|
+
end
|
|
36
|
+
private
|
|
37
|
+
def read_at(io, size, offset)
|
|
38
|
+
return io.pread(size, offset) if io.respond_to?(:pread)
|
|
39
|
+
io.seek(offset)
|
|
40
|
+
io.read(size) || "".b
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
class Context
|
|
45
|
+
attr_reader :store
|
|
46
|
+
def initialize(store) = (@store = store)
|
|
47
|
+
def time_value(number, _format, references:)
|
|
48
|
+
reference = references.select { |value| value <= number }.max || 1
|
|
49
|
+
(@store.metadata(number)[:timestamp_ns] - @store.metadata(reference)[:timestamp_ns]) / 1e9
|
|
50
|
+
end
|
|
51
|
+
def displayed_predecessor(number, snapshot)
|
|
52
|
+
return number - 1 if snapshot[:unfiltered_predecessor]
|
|
53
|
+
return snapshot[:last_displayed] || 0 if number > snapshot[:limit]
|
|
54
|
+
snapshot[:predecessors] ? snapshot[:predecessors].fetch(number, 0) : number - 1
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def initialize(request)
|
|
59
|
+
@reader = Reader.new(request.fetch(:spool))
|
|
60
|
+
@dissector = Gateway::Dissector.new(verify_checksums: request.fetch(:verify_checksums))
|
|
61
|
+
@analysis = Gateway::Analysis.new(registry: @dissector.registry, **request.fetch(:analysis_options))
|
|
62
|
+
@columns = Core::ColumnStore.new
|
|
63
|
+
@annotations = Core::AnnotationStore.new(request.fetch(:spool), persist: false)
|
|
64
|
+
@catalog = Gateway::FieldCatalog.new
|
|
65
|
+
@seen_fields = @catalog.names.to_set
|
|
66
|
+
@context = Context.new(@reader)
|
|
67
|
+
@next = 1
|
|
68
|
+
end
|
|
69
|
+
def analyze(request)
|
|
70
|
+
first, last = request.values_at(:first, :last)
|
|
71
|
+
raise Vanken::Error, "nonsequential analysis batch" unless first == @next && last >= first && last - first < 256
|
|
72
|
+
@reader.limit, @reader.interfaces = last, request.fetch(:interfaces)
|
|
73
|
+
@packets = []
|
|
74
|
+
summaries, summary_index, details, definitions = +"".b, +"".b, {}, {}
|
|
75
|
+
(first..last).each do |number|
|
|
76
|
+
frame = @reader.read(number)
|
|
77
|
+
begin
|
|
78
|
+
packet = @dissector.dissect(frame)
|
|
79
|
+
@analysis.update(packet)
|
|
80
|
+
columns, annotation, info = packet.columns, packet.annotations, packet.info
|
|
81
|
+
packet.fields.each do |field|
|
|
82
|
+
next unless @seen_fields.add?(field.name)
|
|
83
|
+
definitions[field.name] = {type: field.type, source: :dissect, protocol: field.name.split(".").first}
|
|
84
|
+
end
|
|
85
|
+
if packet.reassembled?
|
|
86
|
+
details[number] = JSON.generate("nodes" => Gateway::DetailBuilder.new.build(packet).map(&:to_h), "info" => info,
|
|
87
|
+
"packet" => Gateway::PacketSnapshot.from_packet(packet).to_h) + "\n"
|
|
88
|
+
end
|
|
89
|
+
rescue StandardError => error
|
|
90
|
+
packet = nil
|
|
91
|
+
columns = {source: "", destination: "", protocol: "Malformed", src_port: -1, dst_port: -1, ip_proto: -1, layers: []}
|
|
92
|
+
annotation = {tcp_stream: -1, seq_rel: -1, ack_rel: -1, analysis_flags: [], expert_max: 3,
|
|
93
|
+
expert_items: [{severity: :error, code: "vanken.internal_error", protocol: "vanken", message: error.message}], extra: {}}
|
|
94
|
+
info = error.message
|
|
95
|
+
end
|
|
96
|
+
@packets << packet
|
|
97
|
+
@columns.append(columns)
|
|
98
|
+
@annotations.append(number, annotation)
|
|
99
|
+
summary_index << [summaries.bytesize].pack("Q<")
|
|
100
|
+
summaries << [info.bytesize].pack("L<") << info
|
|
101
|
+
end
|
|
102
|
+
@next = last + 1
|
|
103
|
+
@catalog.merge(definitions)
|
|
104
|
+
@batch = {first: first, last: last, columns: @columns.drain, annotations: @annotations.drain,
|
|
105
|
+
summaries: summaries, summary_index: summary_index, details: details, fields: definitions}
|
|
106
|
+
@batch.merge(matches: matches(request))
|
|
107
|
+
end
|
|
108
|
+
def matches(request)
|
|
109
|
+
expression = request.fetch(:expression)
|
|
110
|
+
return (@batch[:first]..@batch[:last]).to_a if expression.empty?
|
|
111
|
+
program = Core::DisplayFilter.compile(expression, catalog: @catalog)
|
|
112
|
+
snapshot = request.fetch(:snapshot).dup
|
|
113
|
+
@packets.each_with_index.filter_map do |packet, index|
|
|
114
|
+
number = @batch[:first] + index
|
|
115
|
+
next unless packet
|
|
116
|
+
matched = program.match?(Core::FrameView.new(@context, number, packet: packet, snapshot: snapshot))
|
|
117
|
+
snapshot[:last_displayed] = number if matched && !request[:historical_context]
|
|
118
|
+
number if matched
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
def close
|
|
122
|
+
@analysis.close
|
|
123
|
+
ensure
|
|
124
|
+
@reader.close
|
|
125
|
+
end
|
|
126
|
+
def self.run
|
|
127
|
+
$stdin.binmode
|
|
128
|
+
$stdout.binmode
|
|
129
|
+
worker = nil
|
|
130
|
+
loop do
|
|
131
|
+
request = AnalyzerWire.read($stdin)
|
|
132
|
+
response = case request.fetch(:command)
|
|
133
|
+
when :start
|
|
134
|
+
raise Vanken::Error, "duplicate analysis start" if worker
|
|
135
|
+
worker = new(request)
|
|
136
|
+
{pid: Process.pid}
|
|
137
|
+
when :analyze then worker.analyze(request)
|
|
138
|
+
when :match then {matches: worker.matches(request)}
|
|
139
|
+
when :stop then break
|
|
140
|
+
else raise Vanken::Error, "unknown analysis command"
|
|
141
|
+
end
|
|
142
|
+
AnalyzerWire.write($stdout, response)
|
|
143
|
+
rescue StandardError => error
|
|
144
|
+
AnalyzerWire.write($stdout, {error: "#{error.class}: #{error.message}"})
|
|
145
|
+
break
|
|
146
|
+
end
|
|
147
|
+
ensure
|
|
148
|
+
worker&.close
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
Vanken::Capture::AnalyzerWorker.run if $PROGRAM_NAME == __FILE__
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Vanken
|
|
6
|
+
module Capture
|
|
7
|
+
class ControlProtocol
|
|
8
|
+
TYPES = %w[hello started stats warning error stopped].freeze
|
|
9
|
+
MAX_LINE_BYTES = 65_536
|
|
10
|
+
|
|
11
|
+
def initialize(io)
|
|
12
|
+
@io = io
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def write(type, **fields)
|
|
16
|
+
raise ArgumentError, "unknown control message: #{type}" unless TYPES.include?(type.to_s)
|
|
17
|
+
|
|
18
|
+
@io.write(JSON.generate(fields.merge(v: 1, type: type.to_s)) << "\n")
|
|
19
|
+
@io.flush
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.parse(line)
|
|
23
|
+
return if line.bytesize > MAX_LINE_BYTES
|
|
24
|
+
|
|
25
|
+
value = JSON.parse(line)
|
|
26
|
+
value if value.is_a?(Hash) && value["v"] == 1 && TYPES.include?(value["type"])
|
|
27
|
+
rescue JSON::ParserError, EncodingError
|
|
28
|
+
nil
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
require "json"
|
|
5
|
+
require "set"
|
|
6
|
+
require_relative "../errors"
|
|
7
|
+
require_relative "../core/frame_store"
|
|
8
|
+
require_relative "../core/stores"
|
|
9
|
+
require_relative "../core/display_filter/compiler"
|
|
10
|
+
require_relative "../gateway/dissector"
|
|
11
|
+
|
|
12
|
+
module Vanken
|
|
13
|
+
module Capture
|
|
14
|
+
class FilterWorker
|
|
15
|
+
class Reader
|
|
16
|
+
attr_reader :count
|
|
17
|
+
|
|
18
|
+
def initialize(directory, annotations_json)
|
|
19
|
+
@files = []
|
|
20
|
+
@index = open_file(File.join(directory, "frames.idx"))
|
|
21
|
+
@data = open_file(File.join(directory, "frames.bin"))
|
|
22
|
+
@annotations = open_file(File.join(directory, "annotations.bin"))
|
|
23
|
+
@count = @index.size / Core::FrameStore::RECORD_SIZE
|
|
24
|
+
detail_index = File.join(directory, "reassembled.idx")
|
|
25
|
+
if File.file?(detail_index)
|
|
26
|
+
offsets = open_file(detail_index)
|
|
27
|
+
@detail_offsets = read_exact(offsets, (offsets.size / 16) * 16, 0).unpack("Q<*").each_slice(2).to_h
|
|
28
|
+
@details = open_file(File.join(directory, "details.jsonl"))
|
|
29
|
+
end
|
|
30
|
+
interface_path = File.join(directory, "interfaces.json")
|
|
31
|
+
@interfaces = File.file?(interface_path) ? JSON.parse(File.read(interface_path)) : []
|
|
32
|
+
@sparse = File.file?(annotations_json) ? JSON.parse(File.read(annotations_json)) : {}
|
|
33
|
+
rescue StandardError
|
|
34
|
+
close
|
|
35
|
+
raise
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def metadata(number)
|
|
39
|
+
raise IndexError, "invalid frame number" unless number.is_a?(Integer) && number.between?(1, count)
|
|
40
|
+
|
|
41
|
+
offset, caplen, original, timestamp, linktype, interface, direction =
|
|
42
|
+
read_exact(@index, Core::FrameStore::RECORD_SIZE, (number - 1) * Core::FrameStore::RECORD_SIZE).unpack(Core::FrameStore::RECORD)
|
|
43
|
+
raise IOError, "invalid frame spool record" unless caplen <= 16 << 20 && caplen <= original && offset + caplen <= @data.size && direction <= 2
|
|
44
|
+
|
|
45
|
+
{offset: offset, caplen: caplen, original_length: original, timestamp_ns: timestamp,
|
|
46
|
+
linktype: linktype, interface: interface == 0xffff ? nil : @interfaces.fetch(interface),
|
|
47
|
+
direction: [nil, :in, :out].fetch(direction), number: number}
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def frame(number)
|
|
51
|
+
values = metadata(number)
|
|
52
|
+
bytes = read_exact(@data, values.delete(:caplen), values.delete(:offset))
|
|
53
|
+
Core::Frame.new(**values, bytes: bytes)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def annotation(number)
|
|
57
|
+
stream, seq, ack, flags, severity =
|
|
58
|
+
read_exact(@annotations, Core::AnnotationStore::SIZE, (number - 1) * Core::AnnotationStore::SIZE).unpack(Core::AnnotationStore::FORMAT)
|
|
59
|
+
experts = @sparse.fetch("experts", {}).fetch(number.to_s, [])
|
|
60
|
+
{tcp_stream: stream, seq_rel: seq, ack_rel: ack, expert_max: severity, expert_items: experts,
|
|
61
|
+
analysis_flags: Core::AnnotationStore::FLAGS.each_with_index.filter_map { |name, index| name if flags & (1 << index) != 0 },
|
|
62
|
+
extra: @sparse.fetch("extras", {}).fetch(number.to_s, {})}
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def packet_snapshot(number)
|
|
66
|
+
offset = @detail_offsets && @detail_offsets[number]
|
|
67
|
+
return nil unless offset
|
|
68
|
+
raise IOError, "invalid reassembled packet offset" unless offset < @details.size
|
|
69
|
+
|
|
70
|
+
@details.seek(offset)
|
|
71
|
+
record = JSON.parse(@details.gets || raise(IOError, "truncated reassembled packet snapshot"))
|
|
72
|
+
Gateway::PacketSnapshot.from_h(record.fetch("packet"))
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def close = @files&.each { |io| io.close unless io.closed? }
|
|
76
|
+
|
|
77
|
+
private
|
|
78
|
+
|
|
79
|
+
def open_file(path)
|
|
80
|
+
File.open(path, "rb").tap { |io| @files << io }
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def read_exact(io, length, offset)
|
|
84
|
+
return "".b if length.zero?
|
|
85
|
+
|
|
86
|
+
value = if io.respond_to?(:pread)
|
|
87
|
+
io.pread(length, offset)
|
|
88
|
+
else
|
|
89
|
+
io.seek(offset)
|
|
90
|
+
io.read(length)
|
|
91
|
+
end
|
|
92
|
+
raise IOError, "truncated filter spool" unless value && value.bytesize == length
|
|
93
|
+
|
|
94
|
+
value
|
|
95
|
+
rescue EOFError
|
|
96
|
+
raise IOError, "truncated filter spool"
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
class View
|
|
101
|
+
def initialize(reader, number, dissector, context)
|
|
102
|
+
@reader, @number, @dissector, @context = reader, number, dissector, context
|
|
103
|
+
@metadata = reader.metadata(number)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def layer?(name) = packet.layer?(name)
|
|
107
|
+
def field_type(name) = packet.field_type(name)
|
|
108
|
+
|
|
109
|
+
def values(name)
|
|
110
|
+
case name
|
|
111
|
+
when "frame.number" then [@number]
|
|
112
|
+
when "frame.len" then [@metadata[:original_length]]
|
|
113
|
+
when "frame.cap_len" then [@metadata[:caplen]]
|
|
114
|
+
when "frame.time_epoch" then [@metadata[:timestamp_ns] / 1e9]
|
|
115
|
+
when "frame.time_relative"
|
|
116
|
+
reference = @context[:references].reverse.find { |number, _| number <= @number }&.last || @context[:reference]
|
|
117
|
+
[(@metadata[:timestamp_ns] - reference) / 1e9]
|
|
118
|
+
when "frame.time_delta" then [delta(@number - 1)]
|
|
119
|
+
when "frame.time_delta_displayed" then [delta(@context[:predecessors].fetch(@number.to_s, @number - 1))]
|
|
120
|
+
when "frame.interface_name" then @metadata[:interface] ? [@metadata[:interface]["name"]].compact : []
|
|
121
|
+
when "frame.direction" then @metadata[:direction] ? [@metadata[:direction].to_s] : []
|
|
122
|
+
when "frame.marked" then [@context[:marked].include?(@number)]
|
|
123
|
+
when "frame.ignored" then [@context[:ignored].include?(@number)]
|
|
124
|
+
when "frame.protocols" then [packet.columns[:layers].join(":")]
|
|
125
|
+
when "tcp.stream" then annotation[:tcp_stream] >= 0 ? [annotation[:tcp_stream]] : []
|
|
126
|
+
when "tcp.seq_relative" then annotation[:seq_rel] >= 0 ? [annotation[:seq_rel]] : []
|
|
127
|
+
when "tcp.ack_relative" then annotation[:ack_rel] >= 0 ? [annotation[:ack_rel]] : []
|
|
128
|
+
when "tcp.analysis.flags" then annotation[:analysis_flags].empty? ? [] : [true]
|
|
129
|
+
when "expert.code" then annotation[:expert_items].map { |item| item["code"] }
|
|
130
|
+
when "expert.severity"
|
|
131
|
+
items = annotation[:expert_items].map { |item| item["severity"] }
|
|
132
|
+
items.empty? && annotation[:expert_max] > 0 ? [%w[note warning error].fetch(annotation[:expert_max] - 1)] : items
|
|
133
|
+
else
|
|
134
|
+
if name.start_with?("tcp.analysis.")
|
|
135
|
+
return [true] if annotation[:analysis_flags].include?(name.delete_prefix("tcp.analysis."))
|
|
136
|
+
return Array(annotation[:extra][name]).compact
|
|
137
|
+
end
|
|
138
|
+
packet.values(name)
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
private
|
|
143
|
+
|
|
144
|
+
def packet = @packet ||= @reader.packet_snapshot(@number) || @dissector.dissect(@reader.frame(@number))
|
|
145
|
+
def annotation = @annotation ||= @reader.annotation(@number)
|
|
146
|
+
def delta(previous) = previous && previous > 0 ? (@metadata[:timestamp_ns] - @reader.metadata(previous)[:timestamp_ns]) / 1e9 : 0.0
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def self.call(payload)
|
|
150
|
+
validate(payload)
|
|
151
|
+
reader = Reader.new(payload.fetch("spool"), payload.fetch("annotations_json", File.join(payload.fetch("spool"), "annotations.json")))
|
|
152
|
+
first, last = payload.values_at("from", "to")
|
|
153
|
+
raise ArgumentError, "filter chunk exceeds stored frames" unless last <= reader.count + 1
|
|
154
|
+
|
|
155
|
+
dissector = Gateway::Dissector.new(decode_as: payload.fetch("decode_as", []), plugins: payload.fetch("plugins", []))
|
|
156
|
+
program = Core::DisplayFilter.compile(payload.fetch("expr"), catalog: Gateway::FieldCatalog.new)
|
|
157
|
+
context = {marked: Set.new(payload.fetch("marked", [])), ignored: Set.new(payload.fetch("ignored", [])),
|
|
158
|
+
reference: payload.fetch("time_reference_ns") { reader.count.zero? ? 0 : reader.metadata(1)[:timestamp_ns] },
|
|
159
|
+
references: payload.fetch("time_references_ns", {}).map { |number, timestamp| [Integer(number), timestamp] }.sort_by(&:first),
|
|
160
|
+
predecessors: payload.fetch("displayed_predecessors", {})}
|
|
161
|
+
matches = (first...last).select { |number| program.match?(View.new(reader, number, dissector, context)) }
|
|
162
|
+
{"matches" => matches}
|
|
163
|
+
ensure
|
|
164
|
+
reader&.close
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def self.validate(payload)
|
|
168
|
+
raise ArgumentError, "filter worker requires an object" unless payload.is_a?(Hash)
|
|
169
|
+
raise ArgumentError, "spool and expression must be strings" unless %w[spool expr].all? { |key| payload[key].is_a?(String) }
|
|
170
|
+
first, last = payload.values_at("from", "to")
|
|
171
|
+
raise ArgumentError, "invalid filter chunk" unless first.is_a?(Integer) && last.is_a?(Integer) && first >= 1 && last >= first
|
|
172
|
+
%w[decode_as plugins].each do |key|
|
|
173
|
+
values = payload.fetch(key, [])
|
|
174
|
+
raise ArgumentError, "#{key} must be an array of strings" unless values.is_a?(Array) && values.all? { |value| value.is_a?(String) }
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
private_class_method :validate
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "etc"
|
|
4
|
+
require_relative "../version"
|
|
5
|
+
require_relative "helper_options"
|
|
6
|
+
require_relative "control_protocol"
|
|
7
|
+
require_relative "privileges"
|
|
8
|
+
require_relative "../gateway/live_capture"
|
|
9
|
+
require_relative "../gateway/file_writer"
|
|
10
|
+
|
|
11
|
+
module Vanken
|
|
12
|
+
module Capture
|
|
13
|
+
class HelperMain
|
|
14
|
+
def initialize(stdin: $stdin, stdout: $stdout, stderr: $stderr, env: ENV)
|
|
15
|
+
@stdin, @stdout, @env = stdin, stdout, env
|
|
16
|
+
@control = ControlProtocol.new(stderr)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def run(argv)
|
|
20
|
+
@control.write(:hello, pid: Process.pid, vanken: VERSION, redhound: Gateway::LiveCapture.version, ruby: RUBY_VERSION)
|
|
21
|
+
options = HelperOptions.parse(argv)
|
|
22
|
+
case options.command
|
|
23
|
+
when :list_interfaces then @stdout.puts(JSON.generate(Gateway::Interfaces.list)); return 0
|
|
24
|
+
when :check then return check(options)
|
|
25
|
+
when :version then @stdout.puts("Vanken #{VERSION}"); return 0
|
|
26
|
+
when :help then @stdout.puts("vanken-capture -i INTERFACE [--filter EXPR] [--snaplen N] [--no-promiscuous]\n [--buffer-size BYTES] [--direction in|out|inout] [--backend auto|ring|socket|bpf]\n [--drop-to UID:GID] [--stats-interval SEC] [--flush-interval SEC]\n --list-interfaces | --check [--interface IF] | --version"); return 0
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
capture(options)
|
|
30
|
+
rescue ArgumentError => e
|
|
31
|
+
failure("internal", e.message, 2)
|
|
32
|
+
rescue Gateway::Interfaces::NotFound => e
|
|
33
|
+
failure("interface_not_found", e.message, 2)
|
|
34
|
+
rescue Gateway::LiveCapture::Error => e
|
|
35
|
+
failure(e.code, e.message, e.exit_code)
|
|
36
|
+
rescue Privileges::Error, Errno::EPERM, Errno::EACCES => e
|
|
37
|
+
failure("permission_denied", e.message, 3)
|
|
38
|
+
rescue Vanken::FileError, IOError, Errno::EPIPE, Errno::ENOSPC => e
|
|
39
|
+
failure("write_failed", e.message, 1)
|
|
40
|
+
rescue StandardError => e
|
|
41
|
+
failure("internal", e.message, 1)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def capture(options)
|
|
47
|
+
drop_to = options.drop_to || drop_target
|
|
48
|
+
if Process.euid.zero? && !drop_to
|
|
49
|
+
raise Privileges::Error, "privileged capture requires --drop-to UID:GID or an invoking user identity"
|
|
50
|
+
end
|
|
51
|
+
Gateway::Interfaces.find(options.capture[:interface])
|
|
52
|
+
source = Gateway::LiveCapture.open(**options.capture)
|
|
53
|
+
dropped = drop_to ? Privileges.drop!(*drop_to) : false
|
|
54
|
+
@stdout.binmode
|
|
55
|
+
writer = Gateway::FileWriter.open(@stdout, format: :pcapng, linktype: source.linktype, snaplen: options.capture[:snaplen])
|
|
56
|
+
@stop_reason = nil
|
|
57
|
+
previous_traps = %w[TERM INT].to_h { |signal| [signal, Signal.trap(signal) { @stop_reason ||= "signal" }] }
|
|
58
|
+
monitor = Thread.new do
|
|
59
|
+
Thread.current.report_on_exception = false
|
|
60
|
+
@stdin.read(1) until @stdin.eof?
|
|
61
|
+
@stop_reason ||= "stdin_closed"
|
|
62
|
+
rescue IOError, SystemCallError
|
|
63
|
+
@stop_reason ||= "stdin_closed"
|
|
64
|
+
end
|
|
65
|
+
@control.write(:started, interface: options.capture[:interface], linktype: source.linktype,
|
|
66
|
+
snaplen: options.capture[:snaplen], backend: source.backend, filter: options.capture[:filter], privileges_dropped: dropped)
|
|
67
|
+
flushed_at = stats_at = monotonic
|
|
68
|
+
until @stop_reason || source.stopped?
|
|
69
|
+
packet = source.next_packet(timeout: 0.05)
|
|
70
|
+
writer << packet if packet
|
|
71
|
+
now = monotonic
|
|
72
|
+
if now - flushed_at >= options.flush_interval
|
|
73
|
+
writer.flush
|
|
74
|
+
flushed_at = now
|
|
75
|
+
end
|
|
76
|
+
if now - stats_at >= options.stats_interval
|
|
77
|
+
@control.write(:stats, **source.stats, ts: Time.now.to_f)
|
|
78
|
+
stats_at = now
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
stats = source.stats
|
|
82
|
+
writer.write_stats(stats)
|
|
83
|
+
writer.close
|
|
84
|
+
writer = nil
|
|
85
|
+
@control.write(:stopped, reason: @stop_reason || "source_closed", stats: stats)
|
|
86
|
+
0
|
|
87
|
+
ensure
|
|
88
|
+
monitor&.kill
|
|
89
|
+
monitor&.join
|
|
90
|
+
previous_traps&.each { |signal, handler| Signal.trap(signal, handler) }
|
|
91
|
+
begin
|
|
92
|
+
writer&.close
|
|
93
|
+
ensure
|
|
94
|
+
source&.close
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def check(options)
|
|
99
|
+
interface = options.capture[:interface] || Gateway::Interfaces.list.find { |entry| entry[:up] }&.fetch(:name)
|
|
100
|
+
raise Gateway::Interfaces::NotFound, "no capture interfaces found" unless interface
|
|
101
|
+
|
|
102
|
+
source = Gateway::LiveCapture.open(**options.capture.merge(interface: interface))
|
|
103
|
+
@stdout.puts(JSON.generate(direct: true, interface: interface, backend: source.backend, platform: RUBY_PLATFORM, uid: Process.uid))
|
|
104
|
+
0
|
|
105
|
+
rescue Gateway::LiveCapture::Error => e
|
|
106
|
+
@stdout.puts(JSON.generate(direct: false, interface: interface, platform: RUBY_PLATFORM, uid: Process.uid, code: e.code, message: e.message))
|
|
107
|
+
e.exit_code
|
|
108
|
+
ensure
|
|
109
|
+
source&.close
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def drop_target
|
|
113
|
+
return unless Process.euid.zero?
|
|
114
|
+
return [Process.uid, Process.gid] unless Process.uid.zero?
|
|
115
|
+
|
|
116
|
+
if @env["SUDO_UID"] && @env["SUDO_GID"]
|
|
117
|
+
uid, gid = Integer(@env["SUDO_UID"], 10), Integer(@env["SUDO_GID"], 10)
|
|
118
|
+
elsif @env["PKEXEC_UID"]
|
|
119
|
+
uid = Integer(@env["PKEXEC_UID"], 10)
|
|
120
|
+
gid = Etc.getpwuid(uid).gid
|
|
121
|
+
end
|
|
122
|
+
[uid, gid] if uid&.positive? && gid&.positive?
|
|
123
|
+
rescue ArgumentError
|
|
124
|
+
nil
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def failure(code, message, exit_code)
|
|
128
|
+
@control.write(:error, code: code, message: message, fatal: true)
|
|
129
|
+
exit_code
|
|
130
|
+
rescue IOError, SystemCallError
|
|
131
|
+
exit_code
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
|
|
5
|
+
module Vanken
|
|
6
|
+
module Capture
|
|
7
|
+
class HelperOptions
|
|
8
|
+
attr_reader :command, :capture, :drop_to, :stats_interval, :flush_interval
|
|
9
|
+
|
|
10
|
+
def self.parse(argv)
|
|
11
|
+
new.parse(argv)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def initialize
|
|
15
|
+
@command = :capture
|
|
16
|
+
@capture = {interface: nil, filter: nil, snaplen: 262_144, promiscuous: true,
|
|
17
|
+
buffer_size: 4 * 1024 * 1024, direction: :inout, backend: :auto}
|
|
18
|
+
@stats_interval = 1.0
|
|
19
|
+
@flush_interval = 0.05
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def parse(argv)
|
|
23
|
+
modes = []
|
|
24
|
+
parser = OptionParser.new do |opts|
|
|
25
|
+
opts.on("-i", "--interface IF") { |v| @capture[:interface] = v }
|
|
26
|
+
opts.on("-f", "--filter EXPR") { |v| @capture[:filter] = v }
|
|
27
|
+
opts.on("--snaplen N", Integer) { |v| @capture[:snaplen] = bounded(v, 1..16_777_216, "snaplen") }
|
|
28
|
+
opts.on("--no-promiscuous") { @capture[:promiscuous] = false }
|
|
29
|
+
opts.on("--buffer-size N", Integer) { |v| @capture[:buffer_size] = bounded(v, 65_536..268_435_456, "buffer-size") }
|
|
30
|
+
opts.on("--direction VALUE", %w[in out inout]) { |v| @capture[:direction] = v.to_sym }
|
|
31
|
+
opts.on("--backend VALUE", %w[auto ring socket bpf]) { |v| @capture[:backend] = v.to_sym }
|
|
32
|
+
opts.on("--drop-to UID:GID") do |v|
|
|
33
|
+
raise ArgumentError, "drop-to must be a non-root UID:GID" unless v.match?(/\A[0-9]+:[0-9]+\z/)
|
|
34
|
+
|
|
35
|
+
@drop_to = v.split(":").map { |id| bounded(Integer(id, 10), 1..4_294_967_294, "drop-to") }
|
|
36
|
+
end
|
|
37
|
+
opts.on("--stats-interval SEC", Float) { |v| @stats_interval = bounded(v, 0.01..60, "stats-interval") }
|
|
38
|
+
opts.on("--flush-interval SEC", Float) { |v| @flush_interval = bounded(v, 0.001..1, "flush-interval") }
|
|
39
|
+
opts.on("--list-interfaces") { modes << :list_interfaces }
|
|
40
|
+
opts.on("--check") { modes << :check }
|
|
41
|
+
opts.on("--version") { modes << :version }
|
|
42
|
+
opts.on("-h", "--help") { modes << :help }
|
|
43
|
+
end
|
|
44
|
+
# OptionParser accepts long-option abbreviations by default; privileged inputs must be exact.
|
|
45
|
+
known = %w[--interface --filter --snaplen --no-promiscuous --buffer-size --direction --backend --drop-to --stats-interval --flush-interval --list-interfaces --check --version --help]
|
|
46
|
+
argv.each do |arg|
|
|
47
|
+
raise ArgumentError, "unknown option: #{arg}" if arg.start_with?("--") && !known.include?(arg.split("=", 2).first)
|
|
48
|
+
end
|
|
49
|
+
remaining = parser.parse(argv.dup)
|
|
50
|
+
raise ArgumentError, "unexpected arguments: #{remaining.join(' ')}" unless remaining.empty?
|
|
51
|
+
raise ArgumentError, "conflicting commands" if modes.length > 1
|
|
52
|
+
|
|
53
|
+
@command = modes.first || :capture
|
|
54
|
+
raise ArgumentError, "interface is required" if @command == :capture && @capture[:interface].to_s.empty?
|
|
55
|
+
|
|
56
|
+
self
|
|
57
|
+
rescue OptionParser::ParseError => e
|
|
58
|
+
raise ArgumentError, e.message
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def bounded(value, range, name)
|
|
64
|
+
raise ArgumentError, "#{name} must be in #{range}" unless range.cover?(value)
|
|
65
|
+
|
|
66
|
+
value
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|