redhound 1.0.0 → 2.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +35 -0
  3. data/README.md +97 -31
  4. data/docs/API.md +76 -0
  5. data/docs/FILTERS.md +37 -0
  6. data/docs/MIGRATION.md +23 -0
  7. data/docs/PROTOCOLS.md +45 -0
  8. data/docs/USAGE.md +82 -0
  9. data/docs/VALIDATION.md +59 -0
  10. data/docs/json-schema.json +64 -0
  11. data/exe/redhound +2 -1
  12. data/lib/redhound/analysis/flow_key.rb +53 -0
  13. data/lib/redhound/analysis/flow_table.rb +117 -0
  14. data/lib/redhound/analysis/ip_reassembler.rb +176 -0
  15. data/lib/redhound/analysis/stats.rb +138 -0
  16. data/lib/redhound/analysis/tcp_analysis.rb +43 -0
  17. data/lib/redhound/analysis/tcp_reassembler.rb +124 -0
  18. data/lib/redhound/analysis/tcp_stream.rb +151 -0
  19. data/lib/redhound/analysis.rb +129 -0
  20. data/lib/redhound/capture/bsd/bpf_device.rb +152 -0
  21. data/lib/redhound/capture/bsd/constants.rb +29 -0
  22. data/lib/redhound/capture/file_source.rb +83 -0
  23. data/lib/redhound/capture/interface.rb +66 -0
  24. data/lib/redhound/capture/linktype.rb +30 -0
  25. data/lib/redhound/capture/linux/constants.rb +37 -0
  26. data/lib/redhound/capture/linux/cooked.rb +18 -0
  27. data/lib/redhound/capture/linux/ifreq.rb +42 -0
  28. data/lib/redhound/capture/linux/packet_socket.rb +180 -0
  29. data/lib/redhound/capture/linux/sockaddr_ll.rb +33 -0
  30. data/lib/redhound/capture/linux/tpacket_v3.rb +132 -0
  31. data/lib/redhound/capture/source.rb +79 -0
  32. data/lib/redhound/capture/stats.rb +22 -0
  33. data/lib/redhound/capture.rb +69 -0
  34. data/lib/redhound/cli/command.rb +42 -0
  35. data/lib/redhound/cli/options.rb +133 -0
  36. data/lib/redhound/cli/privileges.rb +29 -0
  37. data/lib/redhound/cli/runner.rb +129 -0
  38. data/lib/redhound/context.rb +20 -0
  39. data/lib/redhound/cursor.rb +69 -0
  40. data/lib/redhound/diagnostic.rb +7 -0
  41. data/lib/redhound/dissector.rb +138 -0
  42. data/lib/redhound/engine.rb +66 -0
  43. data/lib/redhound/errors.rb +38 -0
  44. data/lib/redhound/field.rb +62 -0
  45. data/lib/redhound/file/format.rb +56 -0
  46. data/lib/redhound/file/pcap_reader.rb +43 -0
  47. data/lib/redhound/file/pcap_writer.rb +72 -0
  48. data/lib/redhound/file/pcapng_reader.rb +209 -0
  49. data/lib/redhound/file/pcapng_writer.rb +143 -0
  50. data/lib/redhound/file/rotating_writer.rb +98 -0
  51. data/lib/redhound/filter/analyzer.rb +197 -0
  52. data/lib/redhound/filter/bpf/assembler.rb +121 -0
  53. data/lib/redhound/filter/bpf/disassembler.rb +58 -0
  54. data/lib/redhound/filter/bpf/validator.rb +77 -0
  55. data/lib/redhound/filter/bpf/vm.rb +108 -0
  56. data/lib/redhound/filter/codegen.rb +413 -0
  57. data/lib/redhound/filter/lexer.rb +46 -0
  58. data/lib/redhound/filter/parser.rb +206 -0
  59. data/lib/redhound/filter/program.rb +55 -0
  60. data/lib/redhound/filter.rb +30 -0
  61. data/lib/redhound/layer.rb +75 -0
  62. data/lib/redhound/output/fields.rb +20 -0
  63. data/lib/redhound/output/follow.rb +59 -0
  64. data/lib/redhound/output/hexdump.rb +29 -0
  65. data/lib/redhound/output/json.rb +32 -0
  66. data/lib/redhound/output/summary.rb +59 -0
  67. data/lib/redhound/output/timestamp.rb +34 -0
  68. data/lib/redhound/output/tree.rb +24 -0
  69. data/lib/redhound/packet.rb +63 -0
  70. data/lib/redhound/protocols/arp.rb +39 -0
  71. data/lib/redhound/protocols/checksum.rb +38 -0
  72. data/lib/redhound/protocols/data.rb +21 -0
  73. data/lib/redhound/protocols/dhcp.rb +124 -0
  74. data/lib/redhound/protocols/dns.rb +268 -0
  75. data/lib/redhound/protocols/ethernet.rb +24 -0
  76. data/lib/redhound/protocols/http.rb +183 -0
  77. data/lib/redhound/protocols/icmp.rb +191 -0
  78. data/lib/redhound/protocols/ipv4.rb +56 -0
  79. data/lib/redhound/protocols/ipv6.rb +65 -0
  80. data/lib/redhound/protocols/link.rb +70 -0
  81. data/lib/redhound/protocols/llc.rb +37 -0
  82. data/lib/redhound/protocols/ntp.rb +47 -0
  83. data/lib/redhound/protocols/tcp.rb +80 -0
  84. data/lib/redhound/protocols/tls.rb +218 -0
  85. data/lib/redhound/protocols/tunnel.rb +48 -0
  86. data/lib/redhound/protocols/udp.rb +32 -0
  87. data/lib/redhound/protocols/vlan.rb +23 -0
  88. data/lib/redhound/reader.rb +8 -0
  89. data/lib/redhound/registry.rb +64 -0
  90. data/lib/redhound/stream_dissector.rb +168 -0
  91. data/lib/redhound/util/seq.rb +21 -0
  92. data/lib/redhound/version.rb +2 -1
  93. data/lib/redhound/writer.rb +25 -43
  94. data/lib/redhound.rb +81 -9
  95. data/sig/generated/redhound/analysis/flow_key.rbs +15 -0
  96. data/sig/generated/redhound/analysis/flow_table.rbs +98 -0
  97. data/sig/generated/redhound/analysis/ip_reassembler.rbs +40 -0
  98. data/sig/generated/redhound/analysis/stats.rbs +77 -0
  99. data/sig/generated/redhound/analysis/tcp_analysis.rbs +15 -0
  100. data/sig/generated/redhound/analysis/tcp_reassembler.rbs +27 -0
  101. data/sig/generated/redhound/analysis/tcp_stream.rbs +60 -0
  102. data/sig/generated/redhound/analysis.rbs +38 -0
  103. data/sig/generated/redhound/capture/bsd/bpf_device.rbs +39 -0
  104. data/sig/generated/redhound/capture/bsd/constants.rbs +40 -0
  105. data/sig/generated/redhound/capture/file_source.rbs +34 -0
  106. data/sig/generated/redhound/capture/interface.rbs +53 -0
  107. data/sig/generated/redhound/capture/linktype.rbs +20 -0
  108. data/sig/generated/redhound/capture/linux/constants.rbs +58 -0
  109. data/sig/generated/redhound/capture/linux/cooked.rbs +15 -0
  110. data/sig/generated/redhound/capture/linux/ifreq.rbs +18 -0
  111. data/sig/generated/redhound/capture/linux/packet_socket.rbs +57 -0
  112. data/sig/generated/redhound/capture/linux/sockaddr_ll.rbs +31 -0
  113. data/sig/generated/redhound/capture/linux/tpacket_v3.rbs +35 -0
  114. data/sig/generated/redhound/capture/source.rbs +44 -0
  115. data/sig/generated/redhound/capture/stats.rbs +25 -0
  116. data/sig/generated/redhound/capture.rbs +14 -0
  117. data/sig/generated/redhound/cli/command.rbs +12 -0
  118. data/sig/generated/redhound/cli/options.rbs +25 -0
  119. data/sig/generated/redhound/cli/privileges.rbs +12 -0
  120. data/sig/generated/redhound/cli/runner.rbs +27 -0
  121. data/sig/generated/redhound/context.rbs +27 -0
  122. data/sig/generated/redhound/cursor.rbs +59 -0
  123. data/sig/generated/redhound/diagnostic.rbs +21 -0
  124. data/sig/generated/redhound/dissector.rbs +113 -0
  125. data/sig/generated/redhound/engine.rbs +23 -0
  126. data/sig/generated/redhound/errors.rbs +56 -0
  127. data/sig/generated/redhound/field.rbs +74 -0
  128. data/sig/generated/redhound/file/format.rbs +33 -0
  129. data/sig/generated/redhound/file/pcap_reader.rbs +19 -0
  130. data/sig/generated/redhound/file/pcap_writer.rbs +38 -0
  131. data/sig/generated/redhound/file/pcapng_reader.rbs +54 -0
  132. data/sig/generated/redhound/file/pcapng_writer.rbs +50 -0
  133. data/sig/generated/redhound/file/rotating_writer.rbs +42 -0
  134. data/sig/generated/redhound/filter/analyzer.rbs +42 -0
  135. data/sig/generated/redhound/filter/bpf/assembler.rbs +39 -0
  136. data/sig/generated/redhound/filter/bpf/disassembler.rbs +18 -0
  137. data/sig/generated/redhound/filter/bpf/validator.rbs +30 -0
  138. data/sig/generated/redhound/filter/bpf/vm.rbs +26 -0
  139. data/sig/generated/redhound/filter/codegen.rbs +101 -0
  140. data/sig/generated/redhound/filter/lexer.rbs +15 -0
  141. data/sig/generated/redhound/filter/parser.rbs +65 -0
  142. data/sig/generated/redhound/filter/program.rbs +48 -0
  143. data/sig/generated/redhound/filter.rbs +10 -0
  144. data/sig/generated/redhound/layer.rbs +81 -0
  145. data/sig/generated/redhound/output/fields.rbs +15 -0
  146. data/sig/generated/redhound/output/follow.rbs +21 -0
  147. data/sig/generated/redhound/output/hexdump.rbs +15 -0
  148. data/sig/generated/redhound/output/json.rbs +18 -0
  149. data/sig/generated/redhound/output/summary.rbs +21 -0
  150. data/sig/generated/redhound/output/timestamp.rbs +21 -0
  151. data/sig/generated/redhound/output/tree.rbs +15 -0
  152. data/sig/generated/redhound/packet.rbs +77 -0
  153. data/sig/generated/redhound/protocols/arp.rbs +15 -0
  154. data/sig/generated/redhound/protocols/checksum.rbs +15 -0
  155. data/sig/generated/redhound/protocols/data.rbs +15 -0
  156. data/sig/generated/redhound/protocols/dhcp.rbs +28 -0
  157. data/sig/generated/redhound/protocols/dns.rbs +42 -0
  158. data/sig/generated/redhound/protocols/ethernet.rbs +15 -0
  159. data/sig/generated/redhound/protocols/http.rbs +36 -0
  160. data/sig/generated/redhound/protocols/icmp.rbs +41 -0
  161. data/sig/generated/redhound/protocols/ipv4.rbs +15 -0
  162. data/sig/generated/redhound/protocols/ipv6.rbs +24 -0
  163. data/sig/generated/redhound/protocols/link.rbs +33 -0
  164. data/sig/generated/redhound/protocols/llc.rbs +15 -0
  165. data/sig/generated/redhound/protocols/ntp.rbs +17 -0
  166. data/sig/generated/redhound/protocols/tcp.rbs +25 -0
  167. data/sig/generated/redhound/protocols/tls.rbs +51 -0
  168. data/sig/generated/redhound/protocols/tunnel.rbs +24 -0
  169. data/sig/generated/redhound/protocols/udp.rbs +18 -0
  170. data/sig/generated/redhound/protocols/vlan.rbs +12 -0
  171. data/sig/generated/redhound/reader.rbs +7 -0
  172. data/sig/generated/redhound/registry.rbs +45 -0
  173. data/sig/generated/redhound/stream_dissector.rbs +44 -0
  174. data/sig/generated/redhound/util/seq.rbs +20 -0
  175. data/sig/generated/redhound/version.rbs +1 -0
  176. data/sig/generated/redhound/writer.rbs +6 -19
  177. data/sig/generated/redhound.rbs +18 -0
  178. data/sig/shims/io.rbs +7 -0
  179. data/sig/shims/process.rbs +3 -0
  180. metadata +175 -55
  181. data/CODE_OF_CONDUCT.md +0 -132
  182. data/Dockerfile +0 -7
  183. data/Rakefile +0 -15
  184. data/Steepfile +0 -4
  185. data/docker-compose.yml +0 -9
  186. data/lib/redhound/analyzer.rb +0 -34
  187. data/lib/redhound/builder/packet_mreq.rb +0 -56
  188. data/lib/redhound/builder/socket.rb +0 -35
  189. data/lib/redhound/builder.rb +0 -4
  190. data/lib/redhound/command.rb +0 -67
  191. data/lib/redhound/l2/ether.rb +0 -68
  192. data/lib/redhound/l2/protocol.rb +0 -33
  193. data/lib/redhound/l2.rb +0 -4
  194. data/lib/redhound/l3/arp.rb +0 -114
  195. data/lib/redhound/l3/base.rb +0 -49
  196. data/lib/redhound/l3/ipv4.rb +0 -95
  197. data/lib/redhound/l3/ipv6.rb +0 -69
  198. data/lib/redhound/l3/protocol.rb +0 -173
  199. data/lib/redhound/l3/resolver.rb +0 -30
  200. data/lib/redhound/l3.rb +0 -8
  201. data/lib/redhound/l4/base.rb +0 -31
  202. data/lib/redhound/l4/icmp.rb +0 -71
  203. data/lib/redhound/l4/resolver.rb +0 -28
  204. data/lib/redhound/l4/udp.rb +0 -63
  205. data/lib/redhound/l4.rb +0 -6
  206. data/lib/redhound/receiver.rb +0 -45
  207. data/lib/redhound/resolver.rb +0 -22
  208. data/lib/redhound/source/socket.rb +0 -18
  209. data/lib/redhound/source.rb +0 -3
  210. data/rbs_collection.lock.yaml +0 -20
  211. data/rbs_collection.yaml +0 -17
  212. data/sig/generated/redhound/analyzer.rbs +0 -14
  213. data/sig/generated/redhound/builder/packet_mreq.rbs +0 -39
  214. data/sig/generated/redhound/builder/socket.rbs +0 -24
  215. data/sig/generated/redhound/command.rbs +0 -19
  216. data/sig/generated/redhound/l2/ether.rbs +0 -41
  217. data/sig/generated/redhound/l2/protocol.rbs +0 -15
  218. data/sig/generated/redhound/l3/arp.rbs +0 -57
  219. data/sig/generated/redhound/l3/base.rbs +0 -28
  220. data/sig/generated/redhound/l3/ipv4.rbs +0 -53
  221. data/sig/generated/redhound/l3/ipv6.rbs +0 -38
  222. data/sig/generated/redhound/l3/protocol.rbs +0 -16
  223. data/sig/generated/redhound/l3/resolver.rbs +0 -16
  224. data/sig/generated/redhound/l4/base.rbs +0 -19
  225. data/sig/generated/redhound/l4/icmp.rbs +0 -33
  226. data/sig/generated/redhound/l4/resolver.rbs +0 -16
  227. data/sig/generated/redhound/l4/udp.rbs +0 -36
  228. data/sig/generated/redhound/receiver.rbs +0 -19
  229. data/sig/generated/redhound/resolver.rbs +0 -11
  230. data/sig/generated/redhound/source/socket.rbs +0 -13
@@ -0,0 +1,151 @@
1
+ # rbs_inline: enabled
2
+ # frozen_string_literal: true
3
+
4
+ module Redhound
5
+ # @api private
6
+ module Analysis
7
+ # @api private
8
+ class TcpStream
9
+ attr_reader :base_seq, :expected, :diagnostics, :deliveries, :gap_lengths, :fin
10
+ # @rbs (?max_bytes: Integer) -> void
11
+ def initialize(max_bytes: 1 << 20)
12
+ raise ArgumentError, 'max_bytes must be positive' unless max_bytes.positive?
13
+ @max_bytes, @expected, @history_start = max_bytes, 0, 0
14
+ @highest = 0
15
+ @base_seq = nil # @rbs Integer?
16
+ @history = ''.b
17
+ @queue = [] #: Array[untyped]
18
+ @diagnostics, @deliveries, @gap_lengths = [], [], [] # @rbs untyped
19
+ @fin, @fin_offset = false, nil # @rbs untyped
20
+ end
21
+ # @rbs (Integer sequence, String data, ?syn: bool, ?fin: bool, ?frame: Integer) -> Array[String]
22
+ def push(sequence, data, syn: false, fin: false, frame: 0)
23
+ @diagnostics, @deliveries, @gap_lengths = [], [], []
24
+ unless @base_seq
25
+ @base_seq = sequence
26
+ @expected = syn ? 1 : 0
27
+ @history_start = @expected
28
+ @highest = @expected
29
+ end
30
+ base = @base_seq #: Integer
31
+ start = @expected + Util::Seq.signed_diff(sequence, (base + @expected) % Util::Seq::MOD) + (syn ? 1 : 0)
32
+ @fin_offset = start + data.bytesize if fin
33
+ @diagnostics << :out_of_order if !data.empty? && start >= @expected && start < @highest
34
+ @diagnostics << :lost_segment if (!data.empty? || fin) && start > @highest
35
+ @diagnostics << :retransmission if !data.empty? && start < @expected
36
+ @highest = [@highest, start + data.bytesize + (fin ? 1 : 0)].max
37
+ pieces = [[start, data, frame]]
38
+ if start < @expected
39
+ length = [data.bytesize, @expected - start].min
40
+ compare_overlap(start, data, @history_start, @history)
41
+ pieces = length >= data.bytesize ? Array.new : [[start + length, data.byteslice(length..), frame]] #: Array[untyped]
42
+ end
43
+ @queue.each do |old_start, old_data, _old_frame|
44
+ replacement = [] #: Array[untyped]
45
+ pieces.each do |pos, bytes, origin|
46
+ compare_overlap(pos, bytes, old_start, old_data)
47
+ ending, old_end = pos + bytes.bytesize, old_start + old_data.bytesize
48
+ if pos < old_end && ending > old_start
49
+ replacement << [pos, bytes.byteslice(0, old_start - pos), origin] if pos < old_start
50
+ replacement << [old_end, bytes.byteslice(old_end - pos..), origin] if ending > old_end
51
+ else
52
+ replacement << [pos, bytes, origin]
53
+ end
54
+ end
55
+ pieces = replacement
56
+ end
57
+ if !data.empty? && pieces.empty?
58
+ @diagnostics.delete(:out_of_order)
59
+ @diagnostics.delete(:lost_segment)
60
+ @diagnostics << :retransmission unless @diagnostics.include?(:retransmission)
61
+ end
62
+ pieces.each do |piece|
63
+ next if piece[1].empty?
64
+ index = @queue.bsearch_index { |entry| entry[0] >= piece[0] } || @queue.length
65
+ @queue.insert(index, piece)
66
+ end
67
+ drain
68
+ while bytesize > @max_bytes || (@queue.first && @queue.first[0] - @expected > @max_bytes)
69
+ break if @queue.empty?
70
+ flush_gap(false)
71
+ end
72
+ trim_history
73
+ consume_fin
74
+ @deliveries.filter_map(&:first)
75
+ end
76
+
77
+ # @rbs (Integer start, String data, Integer old_start, String old_data) -> void
78
+ def compare_overlap(start, data, old_start, old_data)
79
+ from = [start, old_start].max
80
+ to = [start + data.bytesize, old_start + old_data.bytesize].min
81
+ return unless from < to
82
+ if data.byteslice(from - start, to - from) != old_data.byteslice(from - old_start, to - from) && !@diagnostics.include?(:overlap_mismatch)
83
+ @diagnostics << :overlap_mismatch
84
+ end
85
+ end
86
+ # @rbs () -> void
87
+ def drain
88
+ while @queue.first && @queue.first[0] == @expected
89
+ pos, data, frame = @queue.shift
90
+ @deliveries << [data, frame]
91
+ @history_start = pos if @history.empty?
92
+ @history << data
93
+ @expected += data.bytesize
94
+ end
95
+ end
96
+ # @rbs (?bool clear) -> Array[String]
97
+ def flush_gap(clear = true)
98
+ @deliveries, @gap_lengths = [], [] if clear
99
+ target = @queue.empty? ? @fin_offset : @queue.first[0]
100
+ return [] unless target
101
+ gap = target - @expected
102
+ if gap.positive?
103
+ @diagnostics << :reassembly_gap
104
+ @gap_lengths << gap
105
+ @deliveries << [nil, gap]
106
+ @expected += gap
107
+ @history.clear
108
+ @history_start = @expected
109
+ end
110
+ drain
111
+ consume_fin
112
+ trim_history
113
+ @deliveries.filter_map(&:first)
114
+ end
115
+ # @rbs () -> void
116
+ def trim_history
117
+ # Previously delivered bytes share the stream's bounded budget with holes.
118
+ limit = [@max_bytes - @queue.sum { |item| item[1].bytesize + 128 }, 0].max
119
+ excess = @history.bytesize - limit
120
+ if excess.positive?
121
+ @history = @history.byteslice(excess..) #: String
122
+ @history_start += excess
123
+ end
124
+ end
125
+ # @rbs () -> Integer
126
+ def bytesize = @history.bytesize + @queue.sum { |item| item[1].bytesize + 128 }
127
+ # @rbs () -> bool
128
+ def pending? = !@queue.empty? || (!@fin && !@fin_offset.nil?)
129
+ # @rbs () -> bool
130
+ def fin_seen? = !@fin_offset.nil?
131
+ # @rbs () -> void
132
+ def consume_fin
133
+ if !@fin && @fin_offset && @expected >= @fin_offset
134
+ @fin = true
135
+ @expected += 1
136
+ end
137
+ end
138
+ # @rbs () -> Integer
139
+ def next_sequence = ((@base_seq || 0) + @expected) % Util::Seq::MOD
140
+ # @rbs () -> void
141
+ def clear_deliveries = @deliveries.clear
142
+ # @rbs () -> void
143
+ def release
144
+ @history.clear
145
+ @queue.clear
146
+ @deliveries.clear
147
+ @gap_lengths.clear
148
+ end
149
+ end
150
+ end
151
+ end
@@ -0,0 +1,129 @@
1
+ # rbs_inline: enabled
2
+ # frozen_string_literal: true
3
+
4
+ require_relative 'util/seq'
5
+ require_relative 'stream_dissector'
6
+ require_relative 'analysis/flow_key'
7
+ require_relative 'analysis/tcp_stream'
8
+ require_relative 'analysis/flow_table'
9
+ require_relative 'analysis/ip_reassembler'
10
+ require_relative 'analysis/tcp_analysis'
11
+ require_relative 'analysis/tcp_reassembler'
12
+ require_relative 'analysis/stats'
13
+ require_relative 'output/follow'
14
+
15
+ module Redhound
16
+ # @api private
17
+ module Analysis
18
+ # @api private
19
+ class Session
20
+ attr_reader :flows, :ip_reassembler, :tcp_reassembler, :statistics
21
+ # @rbs (?stats: Array[String], ?follow: String?, ?registry: Registry, ?max_state_bytes: Integer, ?max_flows: Integer, ?ip_bytes: Integer, ?tcp_bytes: Integer, ?stream_bytes: Integer, ?protocol_streams: bool) -> void
22
+ def initialize(stats: [], follow: nil, registry: Registry.default, max_state_bytes: 256 << 20, max_flows: 100_000,
23
+ ip_bytes: 16 << 20, tcp_bytes: 64 << 20, stream_bytes: 1 << 20, protocol_streams: true)
24
+ raise ArgumentError, 'state limit must be positive' unless max_state_bytes.positive?
25
+ @max_state_bytes, @finished = max_state_bytes, false
26
+ @flows = FlowTable.new(max_flows: max_flows)
27
+ @ip_reassembler = IpReassembler.new(max_bytes: [ip_bytes, max_state_bytes].min)
28
+ @follow = follow ? Output::Follow.new(follow) : nil
29
+ @tcp_reassembler = TcpReassembler.new(registry: registry, max_bytes: [tcp_bytes, max_state_bytes].min,
30
+ stream_bytes: stream_bytes, protocol_streams: protocol_streams, follow: @follow)
31
+ @flows.on_remove = ->(flow) { @tcp_reassembler.finish_flow(flow) }
32
+ specs = stats.uniq
33
+ raise ConfigurationError, 'too many statistics specifications (maximum 64)' if specs.size > 64
34
+ row_limit = [100_000, [((128 << 20) / [specs.size, 1].max) / 1024, 1].max].min
35
+ @statistics = specs.map { |spec| statistic(spec, row_limit) }
36
+ end
37
+ # @rbs (String specification, Integer max_rows) -> untyped
38
+ def statistic(specification, max_rows)
39
+ kind, argument = specification.split(',', 2)
40
+ case kind
41
+ when 'io'
42
+ raise ConfigurationError, 'io statistics requires an interval of at least one nanosecond' unless argument && argument.match?(/\A(?:\d+(?:\.\d*)?|\.\d+)\z/) && argument.to_f.finite? && argument.to_f >= 0.000_000_001
43
+ Stats::Io.new(argument.to_f, max_rows: max_rows)
44
+ when 'conv', 'endpoints'
45
+ raise ConfigurationError, 'statistics type must be eth, ip, ipv6, tcp or udp' unless argument && %w[eth ip ipv6 tcp udp].include?(argument)
46
+ klass = kind == 'conv' ? Stats::Conversations : Stats::Endpoints
47
+ klass.new(argument.to_sym, max_rows: max_rows)
48
+ when 'phs'
49
+ raise ConfigurationError, 'phs does not accept an argument' if argument
50
+ Stats::Hierarchy.new(max_rows: max_rows)
51
+ else raise ConfigurationError, "unknown statistics: #{specification}"
52
+ end
53
+ end
54
+ # @rbs (Packet packet) -> void
55
+ def update(packet)
56
+ return if @finished
57
+ @flows.advance(packet.timestamp_ns)
58
+ @ip_reassembler.advance(packet.timestamp_ns)
59
+ fragment = @ip_reassembler.descriptor(packet)
60
+ analyzed = packet
61
+ if fragment
62
+ boundary = packet.layers.index(fragment[1]) #: Integer
63
+ packet.layers.slice!(boundary + 1, packet.layers.length)
64
+ virtual = @ip_reassembler.update(packet)
65
+ if virtual
66
+ packet.meta[:reassembled_from] = virtual.meta[:reassembled_from]
67
+ packet.meta[:reassembled_packet] = virtual
68
+ analyzed = virtual
69
+ else
70
+ @statistics.each { |stat| stat.update(packet) }
71
+ enforce_limit(packet)
72
+ return
73
+ end
74
+ end
75
+ result = @flows.update(analyzed)
76
+ if result
77
+ flow, direction = result
78
+ @tcp_reassembler.update(analyzed, flow, direction) if flow.key[0] == :tcp
79
+ @flows.account(flow)
80
+ end
81
+ if analyzed != packet
82
+ packet.layers.concat(analyzed.layers.reject { |layer| %i[ipv4 ipv6 ipv6_ext].include?(layer.protocol) })
83
+ packet.layers.last.add(:ip_reassembled_from, 'ip.reassembled_from', packet.meta[:reassembled_from])
84
+ end
85
+ @statistics.each { |stat| stat.update(packet) }
86
+ @tcp_reassembler.enforce_limit(@flows, packet)
87
+ enforce_limit(packet)
88
+ end
89
+ # @rbs (Packet packet) -> void
90
+ def enforce_limit(packet)
91
+ while bytesize > @max_state_bytes
92
+ flow = @flows.evict
93
+ unless flow
94
+ statistic = @statistics.max_by(&:bytesize)
95
+ break unless statistic && statistic.bytesize.positive?
96
+ statistic.evict
97
+ end
98
+ packet.layers.last.diagnose(:warning, :state_evicted, 'analysis state memory limit exceeded')
99
+ end
100
+ end
101
+ # @rbs () -> Integer
102
+ def bytesize = @flows.bytesize + @ip_reassembler.bytesize + @statistics.sum(&:bytesize)
103
+ # @rbs (untyped err) -> void
104
+ def snapshot(err)
105
+ @statistics.each { |stat| stat.format(err) }
106
+ err.puts("#{@flows.evicted} flows evicted; #{@flows.expired} flows expired") if @flows.evicted.positive? || @flows.expired.positive?
107
+ err.puts("#{@ip_reassembler.evicted} fragmented datagrams evicted; #{@ip_reassembler.expired} expired") if @ip_reassembler.evicted.positive? || @ip_reassembler.expired.positive?
108
+ end
109
+ # @rbs (untyped out, untyped err) -> void
110
+ def finish(out, err)
111
+ return if @finished
112
+ @flows.values.each do |flow|
113
+ @tcp_reassembler.finish_flow(flow).each do |layer|
114
+ layer.diagnostics.each do |diagnostic|
115
+ message = diagnostic.message.b.gsub(/[^\x20-\x7e]/n) { |byte| format('\\x%02x', byte.getbyte(0)) }
116
+ err.puts("stream #{flow.id} #{layer.protocol}: #{diagnostic.code}: #{message}")
117
+ end
118
+ end
119
+ @flows.account(flow)
120
+ end
121
+ @follow&.format(out)
122
+ snapshot(err)
123
+ ensure
124
+ @finished = true
125
+ @follow&.close
126
+ end
127
+ end
128
+ end
129
+ end
@@ -0,0 +1,152 @@
1
+ # rbs_inline: enabled
2
+ # frozen_string_literal: true
3
+
4
+ module Redhound
5
+ # @api private
6
+ module Capture
7
+ # @api private
8
+ module Bsd
9
+ # @api private
10
+ class BpfDevice < Source
11
+ attr_reader :linktype, :interfaces
12
+
13
+ # @rbs (interface: String | Integer | Interface, ?snaplen: Integer, ?promiscuous: bool, ?buffer_size: Integer?, ?direction: Symbol, ?filter: String?) -> void
14
+ def initialize(interface:, snaplen: 262_144, promiscuous: true, buffer_size: nil, direction: :inout, filter: nil)
15
+ super()
16
+ raise ArgumentError, 'snaplen must be between 1 and 16 MiB' unless snaplen.between?(1, File::Format::MAX_RECORD)
17
+ raise ArgumentError, 'direction must be in, out or inout' unless %i[in out inout].include?(direction)
18
+
19
+ @interface = interface.is_a?(Interface) ? interface : Interface.find(interface)
20
+ raise UnsupportedPlatform, 'any capture is only available on Linux' if @interface.name == 'any'
21
+
22
+ @snaplen, @direction = snaplen, direction
23
+ @device = open_device
24
+ if buffer_size
25
+ raise ArgumentError, 'buffer size must be positive' unless buffer_size.positive?
26
+ @device.ioctl(Constants::BIOCSBLEN, [buffer_size].pack('I'))
27
+ end
28
+ @device.ioctl(Constants::BIOCSETIF, [@interface.name].pack('a16') + "\0" * 16)
29
+ buffer = "\0" * 4
30
+ @device.ioctl(Constants::BIOCGBLEN, buffer)
31
+ @buffer_size = buffer.unpack1('I')
32
+ @device.ioctl(Constants::BIOCGDLT, buffer)
33
+ dlt = buffer.unpack1('I')
34
+ @linktype = dlt == 12 ? 101 : dlt
35
+ @interfaces = [Interface.new(name: @interface.name, index: @interface.index, flags: @interface.flags,
36
+ linktype: @linktype, snaplen:)]
37
+ attach_filter(filter ? Filter.compile(filter, linktype: @linktype, snaplen:) : nil)
38
+ @device.ioctl(Constants::BIOCPROMISC, 0) if promiscuous && !@interface.loopback?
39
+ @device.ioctl(Constants::BIOCIMMEDIATE, [1].pack('I'))
40
+ configure_direction
41
+ @device.ioctl(Constants::BIOCFLUSH, 0)
42
+ @records, @offset = ''.b, 0
43
+ rescue Errno::EPERM, Errno::EACCES => error
44
+ @device&.close
45
+ raise PermissionDenied, "capture requires access to /dev/bpf*: #{error.message}"
46
+ rescue StandardError
47
+ @device&.close
48
+ raise
49
+ end
50
+
51
+ # @rbs (?timeout: Numeric?) -> Packet?
52
+ def next_packet(timeout: nil)
53
+ deadline = deadline_for(timeout)
54
+ until @stopped || @closed
55
+ if @offset >= @records.bytesize
56
+ return nil unless wait_readable(@device, deadline)
57
+ result = @device.read_nonblock(@buffer_size, exception: false)
58
+ next if result == :wait_readable
59
+ return nil unless result
60
+
61
+ @records, @offset = result, 0
62
+ next if @records.empty?
63
+ end
64
+ raise CaptureError, 'truncated BPF record header' if @offset + 18 > @records.bytesize
65
+
66
+ seconds, micros, caplen, original_length, header_length = @records.unpack('I4S', offset: @offset)
67
+ unless header_length >= 18 && caplen <= original_length && @offset + header_length + caplen <= @records.bytesize
68
+ raise CaptureError, 'invalid BPF record length'
69
+ end
70
+ direction = if @extended_header && header_length >= 20
71
+ (@records.getbyte(@offset + 19) & 1).positive? ? :out : :in
72
+ elsif @direction != :inout
73
+ @direction
74
+ end
75
+ data = @records.byteslice(@offset + header_length, [caplen, @snaplen].min)
76
+ @offset += (header_length + caplen + 3) & ~3
77
+ @number += 1
78
+ @capture_stats.captured += 1
79
+ return Packet.new(data, timestamp_ns: seconds * 1_000_000_000 + micros * 1_000,
80
+ original_length:, linktype: @linktype, interface: @interfaces.first, direction:, number: @number,
81
+ meta: { csum_not_ready: direction == :out })
82
+ end
83
+ nil
84
+ end
85
+
86
+ # @rbs (untyped program) -> void
87
+ def attach_filter(program)
88
+ instructions = program ? program.instructions : [[0x06, 0, 0, @snaplen]] #: Array[[Integer, Integer, Integer, Integer]]
89
+ Filter::BPF::Validator.validate!(instructions)
90
+ raise FilterTooLarge, 'macOS BPF programs are limited to 512 instructions' if instructions.size > 512
91
+ packed = instructions.map { |instruction| instruction.pack('S C C L') }.join
92
+ @device.ioctl(Constants::BIOCSETF, [instructions.size, packed].pack('L x4 P'))
93
+ end
94
+
95
+ # @rbs () -> Stats
96
+ def stats
97
+ unless @closed
98
+ buffer = "\0" * 8
99
+ @device.ioctl(Constants::BIOCGSTATS, buffer)
100
+ @capture_stats.received, @capture_stats.dropped = buffer.unpack('I2')
101
+ end
102
+ @capture_stats
103
+ end
104
+
105
+ # @rbs () -> void
106
+ def close
107
+ return if @closed
108
+
109
+ begin
110
+ stats
111
+ ensure
112
+ super
113
+ @device.close
114
+ end
115
+ end
116
+
117
+ private
118
+
119
+ # @rbs () -> ::File
120
+ def open_device
121
+ 256.times do |index|
122
+ begin
123
+ return ::File.open("/dev/bpf#{index}", ::File::RDWR)
124
+ rescue Errno::EBUSY
125
+ next
126
+ rescue Errno::ENOENT
127
+ break
128
+ end
129
+ end
130
+ raise CaptureError, 'no available BPF device'
131
+ end
132
+
133
+ # @rbs () -> void
134
+ def configure_direction
135
+ begin
136
+ @device.ioctl(Constants::BIOCSDIRECTION, [{ in: 1, out: 2, inout: 3 }.fetch(@direction)].pack('I'))
137
+ rescue Errno::EINVAL, Errno::ENOTTY
138
+ raise UnsupportedPlatform, 'outgoing-only capture is unavailable on this macOS version' if @direction == :out
139
+
140
+ @device.ioctl(Constants::BIOCSSEESENT, [@direction == :in ? 0 : 1].pack('I'))
141
+ end
142
+ begin
143
+ @device.ioctl(Constants::BIOCSEXTHDR, [1].pack('I'))
144
+ @extended_header = true
145
+ rescue Errno::EINVAL, Errno::ENOTTY
146
+ @extended_header = false
147
+ end
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end
@@ -0,0 +1,29 @@
1
+ # rbs_inline: enabled
2
+ # frozen_string_literal: true
3
+
4
+ module Redhound
5
+ # @api private
6
+ module Capture
7
+ # @api private
8
+ module Bsd
9
+ # @api private
10
+ module Constants
11
+ # Verified against the macOS SDK's net/bpf.h on arm64 and 64-bit ioctl layout.
12
+ BIOCSBLEN = 0xc0044266
13
+ BIOCGBLEN = 0x40044266
14
+ BIOCSETF = 0x80104267
15
+ BIOCFLUSH = 0x20004268
16
+ BIOCPROMISC = 0x20004269
17
+ BIOCGDLT = 0x4004426a
18
+ BIOCSETIF = 0x8020426c
19
+ BIOCGSTATS = 0x4008426f
20
+ BIOCIMMEDIATE = 0x80044270
21
+ BIOCSSEESENT = 0x80044277
22
+ # Apple XNU bsd/net/bpf_private.h: extended headers expose direction at byte 19.
23
+ BIOCSEXTHDR = 0x8004427c
24
+ BIOCSDIRECTION = 0x8004428b
25
+ BPF_ALIGNMENT = 4
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,83 @@
1
+ # rbs_inline: enabled
2
+ # frozen_string_literal: true
3
+
4
+ module Redhound
5
+ # @api private
6
+ module Capture
7
+ # @api private
8
+ class FileSource < Source
9
+ attr_reader :reader
10
+
11
+ # @rbs (untyped input, ?filter: String?) -> void
12
+ def initialize(input, filter: nil)
13
+ super()
14
+ @owned = input.is_a?(String) && input != '-'
15
+ @io = input.is_a?(String) ? (input == '-' ? $stdin : ::File.open(input, 'rb')) : input
16
+ @io.binmode if @io.respond_to?(:binmode)
17
+ magic = File::Format.read_exact(@io, 4)
18
+ @reader = if magic == File::Format::PCAPNG_MAGIC
19
+ File::PcapngReader.new(@io, magic)
20
+ elsif File::Format::PCAP_MAGICS.key?(magic)
21
+ File::PcapReader.new(@io, magic)
22
+ else
23
+ raise FileFormatError, 'unrecognized capture file magic'
24
+ end
25
+ @filter_expression = filter
26
+ @filters = {} #: Hash[Integer, untyped]
27
+ @filters[@reader.linktype] = Filter.compile(filter, linktype: @reader.linktype) if filter
28
+ rescue StandardError, Interrupt
29
+ @io.close if @owned && @io && !@io.closed?
30
+ raise
31
+ end
32
+
33
+ # @rbs (untyped input, ?filter: String?) -> FileSource
34
+ # @rbs [T] (untyped input, ?filter: String?) { (FileSource) -> T } -> T
35
+ def self.open(input, filter: nil)
36
+ source = new(input, filter:)
37
+ return source unless block_given?
38
+
39
+ begin
40
+ yield source
41
+ ensure
42
+ source.close
43
+ end
44
+ end
45
+
46
+ # @rbs (?timeout: Numeric?) -> Packet?
47
+ def next_packet(timeout: nil)
48
+ return nil if @stopped || @closed
49
+
50
+ while (packet = @reader.next_packet)
51
+ @capture_stats.received += 1
52
+ if @filter_expression
53
+ program = (@filters[packet.linktype] ||= Filter.compile(@filter_expression, linktype: packet.linktype))
54
+ next unless program.match?(packet)
55
+ end
56
+ @capture_stats.captured += 1
57
+ return packet
58
+ end
59
+ stop
60
+ nil
61
+ end
62
+
63
+ # @rbs () -> Integer
64
+ def linktype = @reader.linktype
65
+
66
+ # @rbs () -> Array[Interface]
67
+ def interfaces = @reader.interfaces
68
+
69
+ # @rbs () -> Stats
70
+ def stats
71
+ @reader.respond_to?(:statistics?) && @reader.statistics? ? @reader.stats : super
72
+ end
73
+
74
+ # @rbs () -> void
75
+ def close
76
+ return if @closed
77
+
78
+ super
79
+ @io.close if @owned
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,66 @@
1
+ # rbs_inline: enabled
2
+ # frozen_string_literal: true
3
+
4
+ require 'socket'
5
+
6
+ module Redhound
7
+ # @api private
8
+ module Capture
9
+ # @api private
10
+ class Interface
11
+ attr_reader :name, :index, :linktype, :snaplen, :flags, :mtu, :mac, :description, :filter, :meta
12
+
13
+ # @rbs (name: String, ?index: Integer, ?linktype: Integer, ?snaplen: Integer, ?flags: Integer, ?mtu: Integer?, ?mac: String?, ?description: String?, ?filter: String?, ?meta: Hash[Symbol, untyped]) -> void
14
+ def initialize(name:, index: 0, linktype: 1, snaplen: 262_144, flags: 0, mtu: nil, mac: nil,
15
+ description: nil, filter: nil, meta: {})
16
+ @name, @index, @linktype, @snaplen, @flags = name, index, linktype, snaplen, flags
17
+ @mtu, @mac, @description, @filter, @meta = mtu, mac, description, filter, meta
18
+ end
19
+
20
+ # @rbs () -> Integer
21
+ def ifindex = @index
22
+
23
+ # @rbs () -> bool
24
+ def up? = (@flags & 1).positive?
25
+
26
+ # @rbs () -> bool
27
+ def loopback? = (@flags & 8).positive?
28
+
29
+ # @rbs () -> bool
30
+ def running? = (@flags & 0x40).positive?
31
+
32
+ # @rbs () -> String
33
+ def to_s
34
+ state = [up? ? 'Up' : 'Down', running? ? 'Running' : nil, loopback? ? 'Loopback' : nil].compact.join(', ')
35
+ "#{@index}.#{@name} [#{state}]#{@mac ? " #{@mac}" : ''}#{@mtu ? " mtu #{@mtu}" : ''}"
36
+ end
37
+
38
+ # @rbs () -> Array[Interface]
39
+ def self.all
40
+ Socket.getifaddrs.uniq(&:name).map do |address|
41
+ index = address.ifindex || 0
42
+ interface = new(name: address.name, index:, flags: address.flags)
43
+ if RUBY_PLATFORM.include?('linux')
44
+ Linux::Ifreq.interface(address.name, index:)
45
+ else
46
+ interface
47
+ end
48
+ end.sort_by(&:index)
49
+ end
50
+
51
+ # @rbs (String | Integer value) -> Interface
52
+ def self.find(value)
53
+ return new(name: 'any', linktype: 276) if value.to_s == 'any'
54
+
55
+ found = all.find { |item| item.name == value.to_s || item.index.to_s == value.to_s }
56
+ return found if found
57
+
58
+ names = all.map(&:name)
59
+ suggestions = names.select { |name| name.include?(value.to_s[0, 2]) }
60
+ message = "interface #{value.inspect} not found"
61
+ message += "; available: #{(suggestions.empty? ? names : suggestions).join(', ')}"
62
+ raise InterfaceNotFound, message
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,30 @@
1
+ # rbs_inline: enabled
2
+ # frozen_string_literal: true
3
+
4
+ module Redhound
5
+ # @api private
6
+ module Capture
7
+ # @api private
8
+ module Linktype
9
+ VALUES = { null: 0, ethernet: 1, ether: 1, raw: 101, loop: 108, sll: 113,
10
+ linux_sll: 113, radiotap: 127, ipv4: 228, ipv6: 229, sll2: 276, linux_sll2: 276 }.freeze
11
+
12
+ # @rbs (Integer | Symbol | String value) -> Integer
13
+ def self.resolve(value)
14
+ return value if value.is_a?(Integer) && value.between?(0, 65_535)
15
+
16
+ VALUES.fetch(value.to_s.downcase.to_sym) { raise ArgumentError, "unknown link type: #{value}" }
17
+ end
18
+
19
+ # @rbs (Integer value) -> String
20
+ def self.name(value)
21
+ VALUES.key(value)&.to_s || "LINKTYPE_#{value}"
22
+ end
23
+
24
+ # @rbs (Integer hardware_type) -> Integer
25
+ def self.for_hardware(hardware_type)
26
+ { 1 => 1, 772 => 1, 65_534 => 101, 803 => 127 }.fetch(hardware_type, 276)
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,37 @@
1
+ # rbs_inline: enabled
2
+ # frozen_string_literal: true
3
+
4
+ module Redhound
5
+ # @api private
6
+ module Capture
7
+ # @api private
8
+ module Linux
9
+ # @api private
10
+ module Constants
11
+ AF_PACKET = 17
12
+ SOL_PACKET = 263
13
+ ETH_P_ALL = 3
14
+ PACKET_ADD_MEMBERSHIP = 1
15
+ PACKET_RX_RING = 5
16
+ PACKET_STATISTICS = 6
17
+ PACKET_VERSION = 10
18
+ PACKET_IGNORE_OUTGOING = 23
19
+ PACKET_MR_PROMISC = 1
20
+ PACKET_OUTGOING = 4
21
+ SO_ATTACH_FILTER = 26
22
+ SO_RCVBUFFORCE = 33
23
+ TPACKET_V3 = 2
24
+ SIOCGSTAMPNS = 0x8907
25
+ SIOCGIFHWADDR = 0x8927
26
+ SIOCGIFMTU = 0x8921
27
+ SIOCGIFFLAGS = 0x8913
28
+ TP_STATUS_USER = 0x01
29
+ TP_STATUS_CSUMNOTREADY = 0x08
30
+ TP_STATUS_VLAN_VALID = 0x10
31
+ TP_STATUS_VLAN_TPID_VALID = 0x40
32
+ TP_STATUS_CSUM_VALID = 0x80
33
+ RECEIVE_SIZE = 262_144
34
+ end
35
+ end
36
+ end
37
+ end