vanken 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -0
- data/README.md +26 -11
- data/data/coloring_rules.yml +67 -0
- data/docs/performance.md +10 -0
- data/docs/releases.md +2 -4
- data/docs/usage.md +51 -0
- data/lib/vanken/app/capture_controller.rb +66 -1
- data/lib/vanken/app/document.rb +102 -39
- data/lib/vanken/app/document_analysis.rb +75 -0
- data/lib/vanken/app/document_jobs.rb +52 -9
- data/lib/vanken/app/expert_info.rb +26 -0
- data/lib/vanken/app/io_graph.rb +47 -0
- data/lib/vanken/app/navigation.rb +77 -0
- data/lib/vanken/app/search_job.rb +62 -0
- data/lib/vanken/app/stats_job.rb +45 -0
- data/lib/vanken/capture/analyzer.rb +3 -1
- data/lib/vanken/capture/analyzer_process.rb +9 -6
- data/lib/vanken/capture/analyzer_worker.rb +2 -2
- data/lib/vanken/capture/filter_worker.rb +8 -2
- data/lib/vanken/capture/helper_main.rb +10 -3
- data/lib/vanken/capture/helper_options.rb +6 -2
- data/lib/vanken/capture/launcher.rb +1 -0
- data/lib/vanken/config/analysis_settings.rb +58 -0
- data/lib/vanken/config/columns.rb +138 -0
- data/lib/vanken/config/messages.rb +105 -0
- data/lib/vanken/config/preferences.rb +34 -8
- data/lib/vanken/config/profiles.rb +74 -0
- data/lib/vanken/config/sessions.rb +69 -0
- data/lib/vanken/config/yaml_file.rb +36 -0
- data/lib/vanken/core/coloring.rb +86 -0
- data/lib/vanken/core/frame_store.rb +2 -2
- data/lib/vanken/core/stores.rb +12 -0
- data/lib/vanken/gateway/detail_builder.rb +1 -1
- data/lib/vanken/gateway/display_capture_filter.rb +102 -0
- data/lib/vanken/gateway/dissector.rb +31 -6
- data/lib/vanken/gateway/exporter.rb +125 -0
- data/lib/vanken/gateway/file_writer.rb +26 -2
- data/lib/vanken/gateway/interfaces.rb +59 -0
- data/lib/vanken/gateway/resolver.rb +131 -0
- data/lib/vanken/gateway/statistics.rb +83 -0
- data/lib/vanken/gateway/stream.rb +159 -0
- data/lib/vanken/ui/actions.rb +54 -28
- data/lib/vanken/ui/analysis_dialogs.rb +371 -0
- data/lib/vanken/ui/application.rb +183 -9
- data/lib/vanken/ui/coloring_operations.rb +104 -0
- data/lib/vanken/ui/column_operations.rb +115 -0
- data/lib/vanken/ui/dialogs.rb +41 -26
- data/lib/vanken/ui/file_operations.rb +17 -12
- data/lib/vanken/ui/filter_operations.rb +43 -15
- data/lib/vanken/ui/main_view.rb +74 -29
- data/lib/vanken/ui/navigation_operations.rb +86 -0
- data/lib/vanken/ui/packet_source.rb +33 -1
- data/lib/vanken/ui/selection.rb +8 -2
- data/lib/vanken/ui/settings_operations.rb +233 -0
- data/lib/vanken/version.rb +1 -1
- data/sig/generated/vanken/app/capture_controller.rbs +16 -0
- data/sig/generated/vanken/app/document.rbs +23 -1
- data/sig/generated/vanken/app/document_analysis.rbs +21 -0
- data/sig/generated/vanken/app/document_jobs.rbs +6 -0
- data/sig/generated/vanken/app/expert_info.rbs +30 -0
- data/sig/generated/vanken/app/io_graph.rbs +45 -0
- data/sig/generated/vanken/app/navigation.rbs +31 -0
- data/sig/generated/vanken/app/search_job.rbs +21 -0
- data/sig/generated/vanken/app/stats_job.rbs +18 -0
- data/sig/generated/vanken/capture/helper_options.rbs +6 -0
- data/sig/generated/vanken/config/analysis_settings.rbs +31 -0
- data/sig/generated/vanken/config/columns.rbs +37 -0
- data/sig/generated/vanken/config/messages.rbs +14 -0
- data/sig/generated/vanken/config/preferences.rbs +1 -1
- data/sig/generated/vanken/config/profiles.rbs +29 -0
- data/sig/generated/vanken/config/sessions.rbs +17 -0
- data/sig/generated/vanken/config/yaml_file.rbs +11 -0
- data/sig/generated/vanken/core/coloring.rbs +52 -0
- data/sig/generated/vanken/core/stores.rbs +4 -0
- data/sig/generated/vanken/gateway/display_capture_filter.rbs +37 -0
- data/sig/generated/vanken/gateway/dissector.rbs +15 -2
- data/sig/generated/vanken/gateway/exporter.rbs +27 -0
- data/sig/generated/vanken/gateway/file_writer.rbs +3 -0
- data/sig/generated/vanken/gateway/interfaces.rbs +12 -0
- data/sig/generated/vanken/gateway/resolver.rbs +52 -0
- data/sig/generated/vanken/gateway/statistics.rbs +104 -0
- data/sig/generated/vanken/gateway/stream.rbs +75 -0
- data/sig/generated/vanken/ui/analysis_dialogs.rbs +71 -0
- data/sig/generated/vanken/ui/application.rbs +23 -1
- data/sig/generated/vanken/ui/coloring_operations.rbs +29 -0
- data/sig/generated/vanken/ui/column_operations.rbs +29 -0
- data/sig/generated/vanken/ui/dialogs.rbs +5 -3
- data/sig/generated/vanken/ui/file_operations.rbs +2 -0
- data/sig/generated/vanken/ui/filter_operations.rbs +3 -1
- data/sig/generated/vanken/ui/main_view.rbs +14 -2
- data/sig/generated/vanken/ui/navigation_operations.rbs +43 -0
- data/sig/generated/vanken/ui/packet_source.rbs +4 -0
- data/sig/generated/vanken/ui/settings_operations.rbs +41 -0
- metadata +51 -3
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
require "yaml"
|
|
5
|
+
require "tempfile"
|
|
6
|
+
require "fileutils"
|
|
7
|
+
require_relative "display_filter/compiler"
|
|
8
|
+
|
|
9
|
+
module Vanken
|
|
10
|
+
module Core
|
|
11
|
+
module Coloring
|
|
12
|
+
Rule = Data.define(
|
|
13
|
+
:name, #: String
|
|
14
|
+
:filter, #: String
|
|
15
|
+
:light, #: Hash[String, String]
|
|
16
|
+
:dark, #: Hash[String, String]
|
|
17
|
+
:enabled, #: bool
|
|
18
|
+
:program #: DisplayFilter::Program
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
class RuleSet
|
|
22
|
+
DEFAULT_PATH = File.expand_path("../../../data/coloring_rules.yml", __dir__)
|
|
23
|
+
attr_reader :rules #: Array[Rule]
|
|
24
|
+
|
|
25
|
+
# @rbs (Hash[String, untyped] payload, ?catalog: untyped) -> void
|
|
26
|
+
def initialize(payload, catalog: nil)
|
|
27
|
+
raise Vanken::ConfigError, "unsupported coloring rules schema" unless payload.is_a?(Hash) && payload["schema_version"] == 1 && payload["rules"].is_a?(Array) && payload["rules"].size <= 256
|
|
28
|
+
@rules = payload.fetch("rules").map do |value|
|
|
29
|
+
raise Vanken::ConfigError, "invalid coloring rule" unless value.is_a?(Hash) && value["name"].is_a?(String) && value["filter"].is_a?(String) && !value["filter"].empty? && [true, false].include?(value["enabled"])
|
|
30
|
+
palettes = %w[light dark].map do |theme|
|
|
31
|
+
colors = value[theme]
|
|
32
|
+
raise Vanken::ConfigError, "invalid rule colors" unless colors.is_a?(Hash) && %w[fg bg].all? { |key| colors[key].is_a?(String) && /\A#[\da-fA-F]{6}\z/.match?(colors[key]) }
|
|
33
|
+
colors.slice("fg", "bg").transform_values { |color| color.dup.freeze }.freeze
|
|
34
|
+
end
|
|
35
|
+
Rule.new(value["name"].dup.freeze, value["filter"].dup.freeze, palettes[0], palettes[1], value["enabled"], DisplayFilter.compile(value["filter"], catalog: catalog))
|
|
36
|
+
end.freeze
|
|
37
|
+
rescue DisplayFilter::SyntaxError => error
|
|
38
|
+
raise Vanken::ConfigError, "invalid coloring filter: #{error.message}"
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# @rbs (_FilterView view, ?theme: Symbol) -> Hash[Symbol, String]?
|
|
42
|
+
def match(view, theme: :light)
|
|
43
|
+
rule = @rules.find { |item| item.enabled && item.program.match?(view) }
|
|
44
|
+
return nil unless rule
|
|
45
|
+
palette = theme == :light ? rule.light : rule.dark
|
|
46
|
+
colors = {foreground: palette.fetch("fg")}
|
|
47
|
+
colors[:background] = palette.fetch("bg") unless theme == :high_contrast
|
|
48
|
+
colors
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# @rbs () -> Hash[String, untyped]
|
|
52
|
+
def to_h
|
|
53
|
+
{"schema_version" => 1, "rules" => @rules.map do |rule|
|
|
54
|
+
{"name" => rule.name, "filter" => rule.filter, "light" => rule.light.dup, "dark" => rule.dark.dup, "enabled" => rule.enabled}
|
|
55
|
+
end}
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# @rbs (String path) -> void
|
|
59
|
+
def save(path)
|
|
60
|
+
directory = File.dirname(File.expand_path(path))
|
|
61
|
+
FileUtils.mkdir_p(directory, mode: 0o700)
|
|
62
|
+
temporary = Tempfile.create(["coloring-", ".tmp"], directory)
|
|
63
|
+
temporary.write(YAML.dump(to_h))
|
|
64
|
+
temporary.flush
|
|
65
|
+
temporary.fsync
|
|
66
|
+
temporary.close
|
|
67
|
+
File.rename(temporary.path, path)
|
|
68
|
+
ensure
|
|
69
|
+
temporary&.close unless temporary&.closed?
|
|
70
|
+
File.unlink(temporary.path) if temporary && File.exist?(temporary.path)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# @rbs (String path, ?catalog: untyped) -> RuleSet
|
|
74
|
+
def self.load(path, catalog: nil)
|
|
75
|
+
raise Vanken::ConfigError, "coloring rules file exceeds 1 MiB" if File.size(path) > 1 << 20
|
|
76
|
+
new(YAML.safe_load(File.read(path), permitted_classes: [], aliases: false), catalog: catalog)
|
|
77
|
+
rescue Psych::Exception, SystemCallError => error
|
|
78
|
+
raise Vanken::ConfigError, error.message
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# @rbs (?catalog: untyped) -> RuleSet
|
|
82
|
+
def self.defaults(catalog: nil) = load(DEFAULT_PATH, catalog: catalog)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -114,8 +114,8 @@ module Vanken
|
|
|
114
114
|
[@data, @records, @reader, @index_reader].compact.each(&:close)
|
|
115
115
|
@closed = true
|
|
116
116
|
end
|
|
117
|
+
FileUtils.remove_entry_secure(@directory) if remove && File.directory?(@directory)
|
|
117
118
|
end
|
|
118
|
-
FileUtils.remove_entry_secure(@directory) if remove && File.directory?(@directory)
|
|
119
119
|
end
|
|
120
120
|
|
|
121
121
|
private
|
|
@@ -130,7 +130,7 @@ module Vanken
|
|
|
130
130
|
|
|
131
131
|
def validate_directory
|
|
132
132
|
raise Vanken::FileError, "unsafe session directory" unless File.directory?(@directory) && !File.symlink?(@directory) && File.stat(@directory).uid == Process.uid
|
|
133
|
-
|
|
133
|
+
Dir.children(@directory).each do |name|
|
|
134
134
|
path = File.join(@directory, name)
|
|
135
135
|
next unless File.exist?(path) || File.symlink?(path)
|
|
136
136
|
stat = File.lstat(path)
|
data/lib/vanken/core/stores.rb
CHANGED
|
@@ -116,12 +116,14 @@ module Vanken
|
|
|
116
116
|
SIZE = 32
|
|
117
117
|
attr_reader :streams #: Hash[Integer, Array[Integer]]
|
|
118
118
|
attr_reader :experts #: Hash[Integer, Array[Hash[Symbol, untyped]]]
|
|
119
|
+
attr_reader :expert_count, :expert_max #: Integer
|
|
119
120
|
# @rbs! @extra: Hash[Integer, Hash[String, untyped]]
|
|
120
121
|
# @rbs! @io: File?
|
|
121
122
|
# @rbs (String directory, ?persist: bool) -> void
|
|
122
123
|
def initialize(directory, persist: true)
|
|
123
124
|
@data = +"".b
|
|
124
125
|
@extra, @streams, @experts = {}, Hash.new { |hash, key| hash[key] = [] }, {}
|
|
126
|
+
@expert_count = @expert_max = 0
|
|
125
127
|
@io = File.open(File.join(directory, "annotations.bin"), "wb", 0o600) if persist
|
|
126
128
|
end
|
|
127
129
|
# @rbs (Integer number, annotation_values annotation) -> void
|
|
@@ -132,6 +134,8 @@ module Vanken
|
|
|
132
134
|
@io&.write(record)
|
|
133
135
|
@streams[annotation[:tcp_stream]] << number if annotation[:tcp_stream] >= 0
|
|
134
136
|
@experts[number] = annotation[:expert_items] unless annotation[:expert_items].empty?
|
|
137
|
+
@expert_count += annotation[:expert_items].size
|
|
138
|
+
@expert_max = [@expert_max, annotation[:expert_max]].max
|
|
135
139
|
@extra[number] = annotation[:extra] unless annotation[:extra].empty?
|
|
136
140
|
end
|
|
137
141
|
# @rbs () -> Hash[Symbol, untyped]
|
|
@@ -139,6 +143,7 @@ module Vanken
|
|
|
139
143
|
batch = {data: @data, extras: @extra, streams: @streams.transform_values { |numbers| numbers }, experts: @experts}
|
|
140
144
|
@data = +"".b
|
|
141
145
|
@extra, @streams, @experts = {}, Hash.new { |hash, key| hash[key] = [] }, {}
|
|
146
|
+
@expert_count = @expert_max = 0
|
|
142
147
|
batch
|
|
143
148
|
end
|
|
144
149
|
# @rbs (Hash[Symbol, untyped] batch) -> void
|
|
@@ -148,6 +153,13 @@ module Vanken
|
|
|
148
153
|
@data << batch[:data]
|
|
149
154
|
@extra.merge!(batch[:extras])
|
|
150
155
|
@experts.merge!(batch[:experts])
|
|
156
|
+
batch[:experts].each_value do |items|
|
|
157
|
+
@expert_count += items.size
|
|
158
|
+
items.each do |item|
|
|
159
|
+
rank = {"note" => 1, "warn" => 2, "warning" => 2, "error" => 3}.fetch(item[:severity].to_s, 0)
|
|
160
|
+
@expert_max = [@expert_max, rank].max
|
|
161
|
+
end
|
|
162
|
+
end
|
|
151
163
|
batch[:streams].each { |stream, numbers| @streams[stream].concat(numbers) }
|
|
152
164
|
end
|
|
153
165
|
# @rbs (Integer number) -> annotation_values
|
|
@@ -12,7 +12,7 @@ module Vanken
|
|
|
12
12
|
frame = node("frame", "Frame #{raw.number}: #{raw.original_length} bytes on wire, #{raw.caplen} captured", offset: 0, length: raw.caplen)
|
|
13
13
|
nodes = packet.layers.each_with_index.map do |layer, index|
|
|
14
14
|
prefix = packet.layers.count { |item| item.protocol == layer.protocol } > 1 ? "#{layer.protocol}[#{index}]" : layer.protocol.to_s
|
|
15
|
-
klass =
|
|
15
|
+
klass = packet.registry.protocols[layer.protocol]
|
|
16
16
|
source = packet.layer_source(layer)
|
|
17
17
|
children = layer.fields.map do |field|
|
|
18
18
|
value = field.value
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "capture_filter"
|
|
4
|
+
require_relative "../core/display_filter/parser"
|
|
5
|
+
|
|
6
|
+
module Vanken
|
|
7
|
+
module Gateway
|
|
8
|
+
class DisplayCaptureFilter
|
|
9
|
+
Packet = Struct.new(:data, :original_length, :linktype, :meta)
|
|
10
|
+
TUNNEL_PORTS = [4789, 8472, 6081].freeze
|
|
11
|
+
attr_reader :expression
|
|
12
|
+
|
|
13
|
+
def initialize(expression)
|
|
14
|
+
@expression = convert(Core::DisplayFilter::Parser.new(expression).parse)
|
|
15
|
+
@program = CaptureFilter.compile(@expression) if @expression
|
|
16
|
+
rescue CaptureFilter::Error
|
|
17
|
+
@expression = @program = nil
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# nil requests the ordinary VDF evaluator; false is a definite non-match.
|
|
21
|
+
def match(frame)
|
|
22
|
+
return nil unless @program && eligible?(frame)
|
|
23
|
+
@program.match?(Packet.new(frame.bytes, frame.original_length, frame.linktype, {}))
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def convert(node)
|
|
29
|
+
case node.kind
|
|
30
|
+
when :and, :or
|
|
31
|
+
left, right = convert(node.left), convert(node.right)
|
|
32
|
+
"(#{left}) #{node.kind} (#{right})" if left && right
|
|
33
|
+
when :not
|
|
34
|
+
child = convert(node.left)
|
|
35
|
+
"not (#{child})" if child
|
|
36
|
+
when :test
|
|
37
|
+
field, predicate = node.left.value, node.right
|
|
38
|
+
return {"ip" => "ip", "ipv6" => "ip6", "tcp" => "tcp", "udp" => "udp"}[field] unless predicate
|
|
39
|
+
return nil unless %i[eq ne in].include?(predicate.operator)
|
|
40
|
+
members = predicate.values.map { |member| term(field, member) }
|
|
41
|
+
return nil if members.any?(&:nil?) || members.empty?
|
|
42
|
+
result = members.map { |member| "(#{member})" }.join(" or ")
|
|
43
|
+
predicate.operator == :ne ? "not (#{result})" : result
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def term(field, member)
|
|
48
|
+
token = member.first
|
|
49
|
+
if (port = /\A(tcp|udp)\.(port|srcport|dstport)\z/.match(field))
|
|
50
|
+
return nil unless token.type == :integer && token.value.between?(0, 65_535)
|
|
51
|
+
direction = {"srcport" => "src ", "dstport" => "dst "}.fetch(port[2], "")
|
|
52
|
+
if member.last
|
|
53
|
+
last = member.last
|
|
54
|
+
return nil unless last.type == :integer && last.value.between?(token.value, 65_535)
|
|
55
|
+
"#{port[1]} #{direction}portrange #{token.value}-#{last.value}"
|
|
56
|
+
else
|
|
57
|
+
"#{port[1]} #{direction}port #{token.value}"
|
|
58
|
+
end
|
|
59
|
+
elsif (address = /\A(ip|ipv6)\.(addr|src|dst)\z/.match(field))
|
|
60
|
+
return nil unless token.type == :address && !member.last
|
|
61
|
+
value = IPAddr.new(token.value)
|
|
62
|
+
return nil unless value.ipv4? == (address[1] == "ip")
|
|
63
|
+
direction = {"src" => "src ", "dst" => "dst "}.fetch(address[2], "")
|
|
64
|
+
prefix = token.value.split("/", 2)[1]
|
|
65
|
+
literal = prefix ? "net #{value}/#{prefix}" : "host #{value}"
|
|
66
|
+
"#{address[1] == 'ip' ? 'ip' : 'ip6'} #{direction}#{literal}"
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def eligible?(frame)
|
|
71
|
+
bytes = frame.bytes
|
|
72
|
+
return false unless frame.linktype == 1 && frame.original_length == bytes.bytesize && bytes.bytesize >= 34
|
|
73
|
+
type = bytes.unpack1("n", offset: 12)
|
|
74
|
+
if type == 0x0800
|
|
75
|
+
return false unless bytes.getbyte(14) == 0x45 && (bytes.unpack1("n", offset: 20) & 0x3fff).zero?
|
|
76
|
+
length = bytes.unpack1("n", offset: 16)
|
|
77
|
+
return false unless length >= 20 && length + 14 <= bytes.bytesize
|
|
78
|
+
transport, finish, protocol = 34, length + 14, bytes.getbyte(23)
|
|
79
|
+
elsif type == 0x86dd
|
|
80
|
+
return false unless bytes.bytesize >= 54 && bytes.getbyte(14) >> 4 == 6
|
|
81
|
+
length = bytes.unpack1("n", offset: 18)
|
|
82
|
+
return false unless length.positive? && length + 54 <= bytes.bytesize
|
|
83
|
+
transport, finish, protocol = 54, length + 54, bytes.getbyte(20)
|
|
84
|
+
else
|
|
85
|
+
return false
|
|
86
|
+
end
|
|
87
|
+
if protocol == 6
|
|
88
|
+
return false unless finish - transport >= 20
|
|
89
|
+
header = (bytes.getbyte(transport + 12) >> 4) * 4
|
|
90
|
+
header >= 20 && transport + header <= finish
|
|
91
|
+
elsif protocol == 17
|
|
92
|
+
return false unless finish - transport >= 8
|
|
93
|
+
ports = bytes.unpack("n2", offset: transport)
|
|
94
|
+
length = bytes.unpack1("n", offset: transport + 4)
|
|
95
|
+
(ports & TUNNEL_PORTS).empty? && length >= 8 && transport + length <= finish
|
|
96
|
+
else
|
|
97
|
+
false
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -15,12 +15,36 @@ module Vanken
|
|
|
15
15
|
def self.rank(value) = {nil => 0, note: 1, warning: 2, warn: 2, error: 3}.fetch(value, 0)
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
+
# The public plugin DSL registers through Registry.default; keep that lookup local.
|
|
19
|
+
module RegistryScope
|
|
20
|
+
module Default
|
|
21
|
+
# @rbs () -> Redhound::Registry
|
|
22
|
+
def default = Thread.current[:vanken_registry] || super
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# @rbs [T] (Redhound::Registry registry) { () -> T } -> T
|
|
26
|
+
def self.with(registry)
|
|
27
|
+
previous = Thread.current[:vanken_registry]
|
|
28
|
+
Thread.current[:vanken_registry] = registry
|
|
29
|
+
yield
|
|
30
|
+
ensure
|
|
31
|
+
Thread.current[:vanken_registry] = previous
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
Redhound::Registry.singleton_class.prepend(RegistryScope::Default)
|
|
35
|
+
|
|
18
36
|
class Dissector
|
|
19
37
|
attr_reader :registry #: Redhound::Registry
|
|
20
38
|
# @rbs (?verify_checksums: bool, ?decode_as: Array[String], ?plugins: Array[String]) -> void
|
|
21
39
|
def initialize(verify_checksums: false, decode_as: [], plugins: [])
|
|
22
|
-
plugins.each { |path| load File.expand_path(path) }
|
|
23
40
|
@registry = Redhound::Registry.default.copy
|
|
41
|
+
RegistryScope.with(@registry) do
|
|
42
|
+
plugins.each do |path|
|
|
43
|
+
load File.expand_path(path), true
|
|
44
|
+
rescue LoadError, SyntaxError => error
|
|
45
|
+
raise Vanken::ConfigError, "failed to load plugin #{path}: #{error.message}"
|
|
46
|
+
end
|
|
47
|
+
end
|
|
24
48
|
decode_as.each { |rule| @registry.decode_as(rule) }
|
|
25
49
|
@engine = Redhound::Engine.new(registry: @registry, verify_checksums: verify_checksums)
|
|
26
50
|
end
|
|
@@ -45,6 +69,7 @@ module Vanken
|
|
|
45
69
|
|
|
46
70
|
class PacketView
|
|
47
71
|
attr_reader :raw_packet #: Redhound::Packet
|
|
72
|
+
attr_reader :registry #: Redhound::Registry
|
|
48
73
|
# @rbs (Redhound::Packet packet, Redhound::Registry registry) -> void
|
|
49
74
|
def initialize(packet, registry) = (@raw_packet = packet; @registry = registry)
|
|
50
75
|
# @rbs (String name) -> Array[untyped]
|
|
@@ -86,7 +111,7 @@ module Vanken
|
|
|
86
111
|
end
|
|
87
112
|
# @rbs () -> String
|
|
88
113
|
def info
|
|
89
|
-
safe_text(@raw_packet.summary)
|
|
114
|
+
RegistryScope.with(@registry) { safe_text(@raw_packet.summary) }
|
|
90
115
|
end
|
|
91
116
|
# @rbs () -> Core::annotation_values
|
|
92
117
|
def annotations
|
|
@@ -132,12 +157,12 @@ module Vanken
|
|
|
132
157
|
end
|
|
133
158
|
|
|
134
159
|
class FieldCatalog
|
|
135
|
-
# @rbs () -> void
|
|
136
|
-
def initialize
|
|
160
|
+
# @rbs (?registry: Redhound::Registry) -> void
|
|
161
|
+
def initialize(registry: Redhound::Registry.default)
|
|
137
162
|
@mutex = Mutex.new
|
|
138
163
|
@fields = {} #: Hash[String, Hash[Symbol, untyped]]
|
|
139
|
-
@protocols =
|
|
140
|
-
|
|
164
|
+
@protocols = registry.protocols.keys.map { |id| id == :ipv4 ? "ip" : id.to_s }
|
|
165
|
+
registry.protocols.each do |id, klass|
|
|
141
166
|
definitions = Array(klass.compiled_header&.definitions) #: Array[Redhound::FieldDefinition]
|
|
142
167
|
definitions.each do |field|
|
|
143
168
|
@fields[field.name] = {type: field.type, source: :dissect, protocol: id.to_s}.freeze
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
require "csv"
|
|
5
|
+
require "tempfile"
|
|
6
|
+
require_relative "file_writer"
|
|
7
|
+
require_relative "detail_builder"
|
|
8
|
+
|
|
9
|
+
module Vanken
|
|
10
|
+
module Gateway
|
|
11
|
+
class Exporter
|
|
12
|
+
DEFAULT_COLUMNS = %i[number time source destination protocol length info].map { |key| {key: key, label: key.to_s.capitalize} }.freeze
|
|
13
|
+
|
|
14
|
+
def initialize(document) = (@document = document)
|
|
15
|
+
|
|
16
|
+
def numbers(scope: :all, range: "", selected: nil, exclude_ignored: false)
|
|
17
|
+
limit = @document.count
|
|
18
|
+
empty = [] #: Array[Integer]
|
|
19
|
+
values = case scope.to_sym
|
|
20
|
+
when :all then (1..limit).to_a
|
|
21
|
+
when :displayed then @document.display_numbers
|
|
22
|
+
when :selected then selected ? [selected] : empty
|
|
23
|
+
when :marked then @document.marked.to_a
|
|
24
|
+
when :between_marks
|
|
25
|
+
marks = @document.marked.to_a.sort
|
|
26
|
+
marks.empty? ? empty : (marks.first..marks.last).to_a
|
|
27
|
+
when :range then ranges(range, limit)
|
|
28
|
+
else raise ArgumentError, "invalid packet scope"
|
|
29
|
+
end
|
|
30
|
+
raise ArgumentError, "invalid packet number" unless values.all? { |number| number.is_a?(Integer) && number.between?(1, limit) }
|
|
31
|
+
values = values.reject { |number| @document.ignored.include?(number) } if exclude_ignored
|
|
32
|
+
values.sort.uniq
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def write(path, format:, numbers: nil, columns: DEFAULT_COLUMNS, cancelled: nil)
|
|
36
|
+
format = format.to_sym
|
|
37
|
+
raise ArgumentError, "invalid export format" unless %i[pcap pcapng json ndjson text csv].include?(format)
|
|
38
|
+
numbers = (numbers || self.numbers).to_a.sort.uniq
|
|
39
|
+
raise IndexError, "invalid packet number" unless numbers.all? { |number| number.is_a?(Integer) && number.between?(1, @document.count) }
|
|
40
|
+
file = Tempfile.create([".vanken-export-", ".tmp"], File.dirname(File.expand_path(path)))
|
|
41
|
+
begin
|
|
42
|
+
file.binmode
|
|
43
|
+
case format
|
|
44
|
+
when :pcap, :pcapng
|
|
45
|
+
linktype = numbers.empty? ? 1 : @document.store.metadata(numbers.first)[:linktype]
|
|
46
|
+
FileWriter.open(file, format: format, linktype: linktype) do |writer|
|
|
47
|
+
numbers.each { |number| check_cancelled(cancelled); writer << @document.store.read(number) }
|
|
48
|
+
end
|
|
49
|
+
when :json, :ndjson then json(file, numbers, format, cancelled)
|
|
50
|
+
when :text
|
|
51
|
+
numbers.each do |number|
|
|
52
|
+
check_cancelled(cancelled)
|
|
53
|
+
@document.details(number).each { |node| text_node(file, node, 0) }
|
|
54
|
+
file.write("\n")
|
|
55
|
+
end
|
|
56
|
+
when :csv
|
|
57
|
+
csv = CSV.new(file)
|
|
58
|
+
visible = columns.reject { |column| column[:visible] == false }
|
|
59
|
+
csv << visible.map { |column| column[:label] || column[:key].to_s }
|
|
60
|
+
numbers.each { |number| check_cancelled(cancelled); csv << visible.map { |column| csv_value(number, column) } }
|
|
61
|
+
end
|
|
62
|
+
file.flush
|
|
63
|
+
file.fsync
|
|
64
|
+
file.close
|
|
65
|
+
File.rename(file.path, File.expand_path(path))
|
|
66
|
+
ensure
|
|
67
|
+
file.close unless file.closed?
|
|
68
|
+
FileUtils.rm_f(file.path)
|
|
69
|
+
end
|
|
70
|
+
path
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
def ranges(value, limit)
|
|
76
|
+
raise ArgumentError, "packet range is empty" if value.strip.empty?
|
|
77
|
+
value.split(",", -1).flat_map do |part|
|
|
78
|
+
match = /\A\s*(\d+)(?:\s*-\s*(\d*))?\s*\z/.match(part)
|
|
79
|
+
raise ArgumentError, "invalid packet range: #{part}" unless match
|
|
80
|
+
first = match[1].to_i
|
|
81
|
+
last = match[2].nil? ? first : match[2].empty? ? limit : match[2].to_i
|
|
82
|
+
raise ArgumentError, "packet range is outside capture" unless first.between?(1, limit) && last.between?(first, limit)
|
|
83
|
+
(first..last).to_a
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def check_cancelled(cancelled)
|
|
88
|
+
raise Vanken::Error, "operation cancelled" if cancelled&.call || @document.closing?
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def json(file, numbers, format, cancelled)
|
|
92
|
+
dissector = Dissector.new(**@document.analysis_gateway_options)
|
|
93
|
+
analysis = Analysis.new(registry: dissector.registry, **@document.analysis_options)
|
|
94
|
+
selected = numbers.to_set
|
|
95
|
+
written = false
|
|
96
|
+
file.write("[") if format == :json
|
|
97
|
+
# Replay preceding packets so stateful fields and reassembled layers match the document.
|
|
98
|
+
(1..(numbers.last || 0)).each do |number|
|
|
99
|
+
check_cancelled(cancelled)
|
|
100
|
+
packet = dissector.dissect(@document.store.read(number))
|
|
101
|
+
analysis.update(packet)
|
|
102
|
+
next unless selected.include?(number)
|
|
103
|
+
file.write(",") if format == :json && written
|
|
104
|
+
file.write(JSON.generate(packet.to_h))
|
|
105
|
+
file.write("\n") if format == :ndjson
|
|
106
|
+
written = true
|
|
107
|
+
end
|
|
108
|
+
file.write("]\n") if format == :json
|
|
109
|
+
ensure
|
|
110
|
+
analysis&.close
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def text_node(file, node, depth)
|
|
114
|
+
file.write("#{' ' * depth}#{node.label}\n")
|
|
115
|
+
node.children.each { |child| text_node(file, child, depth + 1) }
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def csv_value(number, column)
|
|
119
|
+
return @document.view(number).values(column[:field]).map(&:to_s).join(", ") if column[:field]
|
|
120
|
+
return @document.time_value(number, column[:format] || :relative) if column[:key].to_sym == :time
|
|
121
|
+
@document.row(number)[column[:key].to_sym]
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
@@ -22,12 +22,34 @@ module Vanken
|
|
|
22
22
|
def initialize(output, format: :pcapng, **options)
|
|
23
23
|
@format = format.to_sym
|
|
24
24
|
@interfaces = {} #: Hash[Hash[String, untyped], Redhound::Capture::Interface]
|
|
25
|
-
|
|
25
|
+
rotating = options[:max_bytes] || options[:interval]
|
|
26
|
+
if rotating
|
|
27
|
+
raise ArgumentError, "ring output must be a fixed file path" unless output.is_a?(String) && output != "-" && !output.include?("%")
|
|
28
|
+
count = options.fetch(:file_count, 10)
|
|
29
|
+
raise ArgumentError, "ring file count must be between 1 and 1000" unless count.is_a?(Integer) && count.between?(1, 1000)
|
|
30
|
+
%i[max_bytes interval].each do |key|
|
|
31
|
+
value = options[key]
|
|
32
|
+
raise ArgumentError, "#{key} must be positive" if value && (!value.is_a?(Numeric) || !value.finite? || !value.positive?)
|
|
33
|
+
end
|
|
34
|
+
options = options.merge(file_count: count)
|
|
35
|
+
# The public Writer cycles slots when max_bytes is present, also for time-based rings.
|
|
36
|
+
options[:max_bytes] ||= 9_223_372_036_854_775_807
|
|
37
|
+
extension = File.extname(output)
|
|
38
|
+
stem = output.delete_suffix(extension)
|
|
39
|
+
count.times do |index|
|
|
40
|
+
path = "#{stem}_#{format('%05d', index)}#{extension}"
|
|
41
|
+
next unless File.exist?(path) || File.symlink?(path)
|
|
42
|
+
File.open(path, File::RDWR | File::NOFOLLOW) do |file|
|
|
43
|
+
raise ArgumentError, "ring output must be a regular, unlinked file owned by this user" unless file.stat.file? && file.stat.nlink == 1 && file.stat.uid == Process.uid
|
|
44
|
+
file.chmod(0o600)
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
elsif output.is_a?(String)
|
|
26
48
|
@owned = File.open(output, "wb", 0o600)
|
|
27
49
|
@owned.chmod(0o600)
|
|
28
50
|
end
|
|
29
51
|
@writer = Redhound::Writer.open(@owned || output, format: @format, **options)
|
|
30
|
-
rescue Redhound::Error, IOError, SystemCallError => error
|
|
52
|
+
rescue Redhound::Error, ArgumentError, IOError, SystemCallError => error
|
|
31
53
|
@owned&.close
|
|
32
54
|
raise Vanken::FileError, error.message
|
|
33
55
|
end
|
|
@@ -45,6 +67,8 @@ module Vanken
|
|
|
45
67
|
end
|
|
46
68
|
# @rbs () -> void
|
|
47
69
|
def flush = @writer.flush
|
|
70
|
+
# @rbs () -> Integer
|
|
71
|
+
def bytes_written = @writer.bytes_written
|
|
48
72
|
# @rbs (Redhound::Capture::Stats | Hash[Symbol | String, Integer] stats, ?interface: Redhound::Capture::Interface?) -> void
|
|
49
73
|
def write_stats(stats, interface: nil)
|
|
50
74
|
if stats.is_a?(Hash)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "redhound"
|
|
4
|
+
require "open3"
|
|
4
5
|
|
|
5
6
|
module Vanken
|
|
6
7
|
module Gateway
|
|
@@ -25,6 +26,64 @@ module Vanken
|
|
|
25
26
|
def self.find(name)
|
|
26
27
|
list.find { |interface| interface[:name] == name } || raise(NotFound, "interface #{name.inspect} not found")
|
|
27
28
|
end
|
|
29
|
+
|
|
30
|
+
# @rbs () -> Hash[String, Integer]
|
|
31
|
+
def self.traffic_counters
|
|
32
|
+
if RUBY_PLATFORM.include?("linux")
|
|
33
|
+
parse_linux_counters(File.read("/proc/net/dev"))
|
|
34
|
+
elsif RUBY_PLATFORM.include?("darwin")
|
|
35
|
+
output, _, status = Open3.capture3("/usr/sbin/netstat", "-ibn")
|
|
36
|
+
status.success? ? parse_darwin_counters(output) : {}
|
|
37
|
+
else
|
|
38
|
+
{}
|
|
39
|
+
end
|
|
40
|
+
rescue SystemCallError, IOError
|
|
41
|
+
{}
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# @rbs (String text) -> Hash[String, Integer]
|
|
45
|
+
def self.parse_linux_counters(text)
|
|
46
|
+
result = {} #: Hash[String, Integer]
|
|
47
|
+
text.each_line do |line|
|
|
48
|
+
name, fields = line.split(":", 2)
|
|
49
|
+
next unless name && fields
|
|
50
|
+
values = fields.split
|
|
51
|
+
next unless values.length >= 16 && values[0].match?(/\A\d+\z/) && values[8].match?(/\A\d+\z/)
|
|
52
|
+
result[name.strip] = values[0].to_i + values[8].to_i
|
|
53
|
+
end
|
|
54
|
+
result
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# @rbs (String text) -> Hash[String, Integer]
|
|
58
|
+
def self.parse_darwin_counters(text)
|
|
59
|
+
lines = text.lines
|
|
60
|
+
header = lines.shift.to_s.split
|
|
61
|
+
first, received, sent = header.index("Ipkts"), header.index("Ibytes"), header.index("Obytes")
|
|
62
|
+
return {} unless first && received && sent
|
|
63
|
+
result = {} #: Hash[String, Integer]
|
|
64
|
+
lines.each do |line|
|
|
65
|
+
fields = line.split
|
|
66
|
+
next unless fields[2]&.start_with?("<Link#")
|
|
67
|
+
counters = fields.last(header.length - first)
|
|
68
|
+
incoming, outgoing = counters[received - first], counters[sent - first]
|
|
69
|
+
next unless incoming&.match?(/\A\d+\z/) && outgoing&.match?(/\A\d+\z/)
|
|
70
|
+
result[fields.first.delete_suffix("*")] = incoming.to_i + outgoing.to_i
|
|
71
|
+
end
|
|
72
|
+
result
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# @rbs (?previous: Hash[Symbol, untyped]?, ?history: Hash[String, Array[Integer]], ?now: Integer | Float) -> Hash[Symbol, untyped]
|
|
76
|
+
def self.sample_traffic(previous: nil, history: {}, now: Process.clock_gettime(Process::CLOCK_MONOTONIC))
|
|
77
|
+
counters = traffic_counters
|
|
78
|
+
elapsed = previous ? now.to_f - previous[:time].to_f : 0.0
|
|
79
|
+
rates = counters.to_h do |name, bytes|
|
|
80
|
+
old = previous && previous[:counters][name]
|
|
81
|
+
rate = old && elapsed.positive? ? [(bytes - old).fdiv(elapsed), 0].max.to_i : 0
|
|
82
|
+
[name, rate]
|
|
83
|
+
end
|
|
84
|
+
samples = rates.to_h { |name, rate| [name, ((history[name] || []) + [rate]).last(60)] }
|
|
85
|
+
{previous: {time: now, counters: counters}, rates: rates, history: samples}
|
|
86
|
+
end
|
|
28
87
|
end
|
|
29
88
|
end
|
|
30
89
|
end
|