raindrops-maintained 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. checksums.yaml +7 -0
  2. data/.document +7 -0
  3. data/.gitattributes +4 -0
  4. data/.gitignore +16 -0
  5. data/.manifest +62 -0
  6. data/.olddoc.yml +16 -0
  7. data/COPYING +165 -0
  8. data/GIT-VERSION-FILE +1 -0
  9. data/GIT-VERSION-GEN +40 -0
  10. data/GNUmakefile +4 -0
  11. data/LATEST +9 -0
  12. data/LICENSE +16 -0
  13. data/NEWS +384 -0
  14. data/README +101 -0
  15. data/TODO +3 -0
  16. data/archive/.gitignore +3 -0
  17. data/archive/slrnpull.conf +4 -0
  18. data/examples/linux-listener-stats.rb +122 -0
  19. data/examples/middleware.ru +5 -0
  20. data/examples/watcher.ru +4 -0
  21. data/examples/watcher_demo.ru +13 -0
  22. data/examples/yahns.conf.rb +30 -0
  23. data/examples/zbatery.conf.rb +16 -0
  24. data/ext/raindrops/extconf.rb +163 -0
  25. data/ext/raindrops/linux_inet_diag.c +713 -0
  26. data/ext/raindrops/my_fileno.h +16 -0
  27. data/ext/raindrops/raindrops.c +487 -0
  28. data/ext/raindrops/raindrops_atomic.h +23 -0
  29. data/ext/raindrops/tcp_info.c +245 -0
  30. data/lib/raindrops/aggregate/last_data_recv.rb +94 -0
  31. data/lib/raindrops/aggregate/pmq.rb +245 -0
  32. data/lib/raindrops/aggregate.rb +8 -0
  33. data/lib/raindrops/last_data_recv.rb +102 -0
  34. data/lib/raindrops/linux.rb +77 -0
  35. data/lib/raindrops/middleware/proxy.rb +40 -0
  36. data/lib/raindrops/middleware.rb +153 -0
  37. data/lib/raindrops/struct.rb +62 -0
  38. data/lib/raindrops/watcher.rb +428 -0
  39. data/lib/raindrops.rb +72 -0
  40. data/pkg.mk +151 -0
  41. data/raindrops-maintained.gemspec +1 -0
  42. data/raindrops.gemspec +26 -0
  43. data/setup.rb +1586 -0
  44. data/test/ipv6_enabled.rb +9 -0
  45. data/test/rack_unicorn.rb +11 -0
  46. data/test/test_aggregate_pmq.rb +65 -0
  47. data/test/test_inet_diag_socket.rb +16 -0
  48. data/test/test_last_data_recv.rb +57 -0
  49. data/test/test_last_data_recv_unicorn.rb +69 -0
  50. data/test/test_linux.rb +281 -0
  51. data/test/test_linux_all_tcp_listen_stats.rb +66 -0
  52. data/test/test_linux_all_tcp_listen_stats_leak.rb +43 -0
  53. data/test/test_linux_ipv6.rb +166 -0
  54. data/test/test_linux_middleware.rb +64 -0
  55. data/test/test_linux_reuseport_tcp_listen_stats.rb +51 -0
  56. data/test/test_middleware.rb +128 -0
  57. data/test/test_middleware_unicorn.rb +37 -0
  58. data/test/test_middleware_unicorn_ipv6.rb +37 -0
  59. data/test/test_raindrops.rb +207 -0
  60. data/test/test_raindrops_gc.rb +38 -0
  61. data/test/test_struct.rb +54 -0
  62. data/test/test_tcp_info.rb +88 -0
  63. data/test/test_watcher.rb +186 -0
  64. metadata +193 -0
@@ -0,0 +1,77 @@
1
+ # -*- encoding: binary -*-
2
+
3
+ # For reporting TCP ListenStats, users of older \Linux kernels need to ensure
4
+ # that the the "inet_diag" and "tcp_diag" kernel modules are loaded as they do
5
+ # not autoload correctly. The inet_diag facilities of \Raindrops is useful
6
+ # for periodic snapshot reporting of listen queue sizes.
7
+ #
8
+ # Instead of snapshotting, Raindrops::Aggregate::LastDataRecv may be used
9
+ # to aggregate statistics from +all+ accepted sockets as they arrive
10
+ # based on the +last_data_recv+ field in Raindrops::TCP_Info
11
+
12
+ module Raindrops::Linux
13
+
14
+ # The standard proc path for active UNIX domain sockets, feel free to call
15
+ # String#replace on this if your /proc is mounted in a non-standard location
16
+ # for whatever reason
17
+ PROC_NET_UNIX_ARGS = [ '/proc/net/unix', { encoding: "binary" }]
18
+
19
+ # Get ListenStats from an array of +paths+
20
+ #
21
+ # Socket state mapping from integer => symbol, based on socket_state
22
+ # enum from include/linux/net.h in the \Linux kernel:
23
+ # typedef enum {
24
+ # SS_FREE = 0, /* not allocated */
25
+ # SS_UNCONNECTED, /* unconnected to any socket */
26
+ # SS_CONNECTING, /* in process of connecting */
27
+ # SS_CONNECTED, /* connected to socket */
28
+ # SS_DISCONNECTING /* in process of disconnecting */
29
+ # } socket_state;
30
+ # * SS_CONNECTING maps to ListenStats#queued
31
+ # * SS_CONNECTED maps to ListenStats#active
32
+ #
33
+ # This method may be significantly slower than its tcp_listener_stats
34
+ # counterpart due to the latter being able to use inet_diag via netlink.
35
+ # This parses /proc/net/unix as there is no other (known) way
36
+ # to expose Unix domain socket statistics over netlink.
37
+ def unix_listener_stats(paths = nil)
38
+ rv = Hash.new { |h,k| h[k.freeze] = Raindrops::ListenStats.new(0, 0) }
39
+ if nil == paths
40
+ paths = [ '[^\n]+' ]
41
+ else
42
+ paths = paths.map do |path|
43
+ path = path.dup
44
+ path.force_encoding(Encoding::BINARY)
45
+ if File.symlink?(path)
46
+ link = path
47
+ path = File.readlink(link)
48
+ path.force_encoding(Encoding::BINARY)
49
+ rv[link] = rv[path] # vivify ListenerStats
50
+ else
51
+ rv[path] # vivify ListenerStats
52
+ end
53
+ Regexp.escape(path)
54
+ end
55
+ end
56
+ paths = /^\w+: \d+ \d+ (\d+) \d+ (\d+)\s+\d+ (#{paths.join('|')})$/n
57
+
58
+ # no point in pread since we can't stat for size on this file
59
+ File.read(PROC_NET_UNIX_ARGS[0], encoding: 'binary').scan(paths) do |s|
60
+ path = s[-1]
61
+ case s[0]
62
+ when "00000000" # client sockets
63
+ case s[1].to_i
64
+ when 2 then rv[path].queued += 1
65
+ when 3 then rv[path].active += 1
66
+ end
67
+ else
68
+ # listeners, vivify empty stats
69
+ rv[path]
70
+ end
71
+ end
72
+
73
+ rv
74
+ end
75
+ module_function :unix_listener_stats
76
+
77
+ end # Raindrops::Linux
@@ -0,0 +1,40 @@
1
+ # -*- encoding: binary -*-
2
+ # :stopdoc:
3
+ # This class is used by Raindrops::Middleware to proxy application
4
+ # response bodies. There should be no need to use it directly.
5
+ class Raindrops::Middleware::Proxy
6
+ def initialize(body, stats)
7
+ @body, @stats = body, stats
8
+ end
9
+
10
+ # yield to the Rack server here for writing
11
+ def each
12
+ @body.each { |x| yield x }
13
+ end
14
+
15
+ # the Rack server should call this after #each (usually ensure-d)
16
+ def close
17
+ @stats.decr_writing
18
+ @body.close if @body.respond_to?(:close)
19
+ end
20
+
21
+ # Some Rack servers can optimize response processing if it responds
22
+ # to +to_path+ via the sendfile(2) system call, we proxy +to_path+
23
+ # to the underlying body if possible.
24
+ def to_path
25
+ @body.to_path
26
+ end
27
+
28
+ # Rack servers use +respond_to?+ to check for the presence of +close+
29
+ # and +to_path+ methods.
30
+ def respond_to?(m, include_all = false)
31
+ m = m.to_sym
32
+ :close == m || @body.respond_to?(m, include_all)
33
+ end
34
+
35
+ # Avoid breaking users of non-standard extensions (e.g. #body)
36
+ # Rack::BodyProxy does the same.
37
+ def method_missing(*args, &block)
38
+ @body.__send__(*args, &block)
39
+ end
40
+ end
@@ -0,0 +1,153 @@
1
+ # -*- encoding: binary -*-
2
+ require 'raindrops'
3
+ require 'thread'
4
+
5
+ # Raindrops::Middleware is Rack middleware that allows snapshotting
6
+ # current activity from an HTTP request. For all operating systems,
7
+ # it returns at least the following fields:
8
+ #
9
+ # * calling - the number of application dispatchers on your machine
10
+ # * writing - the number of clients being written to on your machine
11
+ #
12
+ # Additional fields are available for \Linux users.
13
+ #
14
+ # It should be loaded at the top of Rack middleware stack before other
15
+ # middlewares for maximum accuracy.
16
+ #
17
+ # === Usage (Rainbows!/Unicorn preload_app=false)
18
+ #
19
+ # If you're using preload_app=false (the default) in your Rainbows!/Unicorn
20
+ # config file, you'll need to create the global Stats object before
21
+ # forking.
22
+ #
23
+ # require 'raindrops'
24
+ # $stats ||= Raindrops::Middleware::Stats.new
25
+ #
26
+ # In your Rack config.ru:
27
+ #
28
+ # use Raindrops::Middleware, :stats => $stats
29
+ #
30
+ # === Usage (Rainbows!/Unicorn preload_app=true)
31
+ #
32
+ # If you're using preload_app=true in your Rainbows!/Unicorn
33
+ # config file, just add the middleware to your stack:
34
+ #
35
+ # In your Rack config.ru:
36
+ #
37
+ # use Raindrops::Middleware
38
+ #
39
+ # === Linux-only extras!
40
+ #
41
+ # To get bound listener statistics under \Linux, you need to specify the
42
+ # listener names for your server. You can even include listen sockets for
43
+ # *other* servers on the same machine. This can be handy for monitoring
44
+ # your nginx proxy as well.
45
+ #
46
+ # In your Rack config.ru, just pass the :listeners argument as an array of
47
+ # strings (along with any other arguments). You can specify any
48
+ # combination of TCP or Unix domain socket names:
49
+ #
50
+ # use Raindrops::Middleware, :listeners => %w(0.0.0.0:80 /tmp/.sock)
51
+ #
52
+ # If you're running Unicorn 0.98.0 or later, you don't have to pass in
53
+ # the :listeners array, Raindrops will automatically detect the listeners
54
+ # used by Unicorn master process. This does not detect listeners in
55
+ # different processes, of course.
56
+ #
57
+ # The response body includes the following stats for each listener
58
+ # (see also Raindrops::ListenStats):
59
+ #
60
+ # * active - total number of active clients on that listener
61
+ # * queued - total number of queued (pre-accept()) clients on that listener
62
+ #
63
+ # = Demo Server
64
+ #
65
+ # There is a server running this middleware (and Watcher) at
66
+ # https://yhbt.net/raindrops-demo/_raindrops
67
+ #
68
+ # Also check out the Watcher demo at https://yhbt.net/raindrops-demo/
69
+ #
70
+ # The demo server is only limited to 30 users, so be sure not to abuse it
71
+ # by using the /tail/ endpoint too much.
72
+ #
73
+ class Raindrops::Middleware
74
+ attr_accessor :app, :stats, :path, :tcp, :unix # :nodoc:
75
+
76
+ # A Raindrops::Struct used to count the number of :calling and :writing
77
+ # clients. This struct is intended to be shared across multiple processes
78
+ # and both counters are updated atomically.
79
+ #
80
+ # This is supported on all operating systems supported by Raindrops
81
+ Stats = Raindrops::Struct.new(:calling, :writing)
82
+
83
+ # :stopdoc:
84
+ require "raindrops/middleware/proxy"
85
+ # :startdoc:
86
+
87
+ # +app+ may be any Rack application, this middleware wraps it.
88
+ # +opts+ is a hash that understands the following members:
89
+ #
90
+ # * :stats - Raindrops::Middleware::Stats struct (default: Stats.new)
91
+ # * :path - HTTP endpoint used for reading the stats (default: "/_raindrops")
92
+ # * :listeners - array of host:port or socket paths (default: from Unicorn)
93
+ def initialize(app, opts = {})
94
+ @app = app
95
+ @stats = opts[:stats] || Stats.new
96
+ @path = opts[:path] || "/_raindrops"
97
+ @mtx = Mutex.new
98
+ tmp = opts[:listeners]
99
+ if tmp.nil? && defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
100
+ tmp = Unicorn.listener_names
101
+ end
102
+ @nl_sock = @tcp = @unix = nil
103
+
104
+ if tmp
105
+ @tcp = tmp.grep(/\A.+:\d+\z/)
106
+ @unix = tmp.grep(%r{\A/})
107
+ @tcp = nil if @tcp.empty?
108
+ @unix = nil if @unix.empty?
109
+ end
110
+ end
111
+
112
+ # standard Rack endpoint
113
+ def call(env) # :nodoc:
114
+ env['PATH_INFO'] == @path and return stats_response
115
+ begin
116
+ @stats.incr_calling
117
+
118
+ status, headers, body = @app.call(env)
119
+ rv = [ status, headers, Proxy.new(body, @stats) ]
120
+
121
+ # the Rack server will start writing headers soon after this method
122
+ @stats.incr_writing
123
+ rv
124
+ ensure
125
+ @stats.decr_calling
126
+ end
127
+ end
128
+
129
+ def stats_response # :nodoc:
130
+ body = "calling: #{@stats.calling}\n" \
131
+ "writing: #{@stats.writing}\n"
132
+
133
+ if defined?(Raindrops::Linux.tcp_listener_stats)
134
+ @mtx.synchronize do
135
+ @nl_sock ||= Raindrops::InetDiagSocket.new
136
+ Raindrops::Linux.tcp_listener_stats(@tcp, @nl_sock).each do |addr,stats|
137
+ body << "#{addr} active: #{stats.active}\n" \
138
+ "#{addr} queued: #{stats.queued}\n"
139
+ end
140
+ end if @tcp
141
+ Raindrops::Linux.unix_listener_stats(@unix).each do |addr,stats|
142
+ body << "#{addr} active: #{stats.active}\n" \
143
+ "#{addr} queued: #{stats.queued}\n"
144
+ end if @unix
145
+ end
146
+
147
+ headers = {
148
+ "Content-Type" => "text/plain",
149
+ "Content-Length" => body.size.to_s,
150
+ }
151
+ [ 200, headers, [ body ] ]
152
+ end
153
+ end
@@ -0,0 +1,62 @@
1
+ # -*- encoding: binary -*-
2
+
3
+ # This is a wrapper around Raindrops objects much like the core Ruby
4
+ # \Struct can be seen as a wrapper around the core \Array class.
5
+ # It's usage is similar to the core \Struct class, except its fields
6
+ # may only be used to house unsigned long integers.
7
+ #
8
+ # class Foo < Raindrops::Struct.new(:readers, :writers)
9
+ # end
10
+ #
11
+ # foo = Foo.new 0, 0
12
+ #
13
+ # foo.incr_writers -> 1
14
+ # foo.incr_readers -> 1
15
+ #
16
+ class Raindrops::Struct
17
+
18
+ # returns a new class derived from Raindrops::Struct and supporting
19
+ # the given +members+ as fields, just like \Struct.new in core Ruby.
20
+ def self.new(*members)
21
+ members = members.map { |x| x.to_sym }.freeze
22
+ str = <<EOS
23
+ def initialize(*values)
24
+ (MEMBERS.size >= values.size) or raise ArgumentError, "too many arguments"
25
+ @raindrops = Raindrops.new(MEMBERS.size)
26
+ values.each_with_index { |val,i| @raindrops[i] = values[i] }
27
+ end
28
+
29
+ def initialize_copy(src)
30
+ @raindrops = src.instance_variable_get(:@raindrops).dup
31
+ end
32
+
33
+ def []=(index, value)
34
+ @raindrops[index] = value
35
+ end
36
+
37
+ def [](index)
38
+ @raindrops[index]
39
+ end
40
+
41
+ def to_hash
42
+ ary = @raindrops.to_ary
43
+ rv = {}
44
+ MEMBERS.each_with_index { |member, i| rv[member] = ary[i] }
45
+ rv
46
+ end
47
+ EOS
48
+
49
+ members.each_with_index do |member, i|
50
+ str << "def incr_#{member}; @raindrops.incr(#{i}); end; " \
51
+ "def decr_#{member}; @raindrops.decr(#{i}); end; " \
52
+ "def #{member}; @raindrops[#{i}]; end; " \
53
+ "def #{member}=(val); @raindrops[#{i}] = val; end; "
54
+ end
55
+
56
+ klass = Class.new
57
+ klass.const_set(:MEMBERS, members)
58
+ klass.class_eval(str)
59
+ klass
60
+ end
61
+
62
+ end