sitegoose 0.3.3

data/lib/sitegoose/resources.rb

@@ -0,0 +1,449 @@
+# frozen_string_literal: true
+
+# How the machine is doing (protocol in sdk/protocols/resources.md).
+#
+# CPU, load, memory, swap and disks, sampled every minute and sent to
+# POST /events/resources every few minutes. The companion of the performance
+# module: that says how the requests went, this says whether the box can
+# still serve them.
+#
+# Nothing is guessed. A reading this machine won't give us is left out of the
+# row, never sent as a zero: the collector stores it as NULL and the dashboard
+# shows "-", because "0% CPU" and "we couldn't measure CPU" are different
+# statements and only one of them means healthy.
+#
+# Linux's /proc for everything but the disks. Ruby's standard library has no
+# statvfs, so the disks come from `df -kP` and `df -iP` (POSIX output, the
+# same statvfs figures: used is total minus free, and total is used plus what
+# is available to us, which is the percentage df prints); without df, the
+# disks are left out.
+
+require "etc"
+require "io/wait"
+
+module SiteGoose
+  class Resources
+    # Keep in step with the collector (expresscollect/Analytics/resources.js) and
+    # the agent (sdk/vps-agent).
+    SAMPLE_EVERY = 60
+    EVERY = 5 * 60
+    RETRY_EVERY = 60
+    OFF_EVERY = 600
+    MAX_ROWS = 500
+    MAX_DISKS = 20
+    MIN_DISK_MB = 1024
+    MIN_CPU_SECONDS = 1.0
+    FSTYPES = %w[ext2 ext3 ext4 xfs btrfs zfs f2fs jfs reiserfs ufs vfat exfat ntfs ntfs3 overlay apfs hfs].freeze
+    MB = 1024 * 1024
+    IO_SECTOR = 512
+    IO_MAX_PAIR_SECONDS = 300
+    IO_NOT_A_DISK = /\A(?:loop|ram|zram|sr|fd|dm-|md\d)/.freeze
+    IO_PARTITION = /\A(?:(?:sd|vd|xvd|hd)[a-z]+\d+|(?:nvme\d+n\d+|mmcblk\d+)p\d+)\z/.freeze
+    NET_NOT_REAL = /\A(?:lo|veth|docker|br-|virbr|tap|tun|vnet)/.freeze
+    CPU_SHARES = [
+      ["cpu_user_pct", [0, 1]],
+      ["cpu_system_pct", [2]],
+      ["cpu_iowait_pct", [4]],
+      ["cpu_steal_pct", [7]]
+    ].freeze
+    DF_TIMEOUT = 5
+
+    class << self
+      def read_text(path, limit = 65_536)
+        File.open(path, "rb") { |f| SiteGoose.text(f.read(limit).to_s) }
+      rescue SystemCallError, IOError
+        ""
+      end
+
+      # The jiffie counters from /proc/stat's first line, or nil.
+      # In order: user, nice, system, idle, iowait, irq, softirq, steal.
+      def cpu_fields(text)
+        text.each_line do |line|
+          next unless line.start_with?("cpu ")
+
+          fields = line.split[1..]
+          return nil unless fields.all? { |f| f.match?(/\A\d+\z/) }
+
+          fields = fields.map(&:to_i)
+          return fields.size >= 4 ? fields : nil
+        end
+        nil
+      end
+
+      # guest and guest_nice (columns 9 and 10) are already inside user and nice.
+      def cpu_total(fields)
+        fields.first(8).sum
+      end
+
+      def cpu_times_of(fields)
+        total = cpu_total(fields)
+        idle = fields[3] + (fields.size > 4 ? fields[4] : 0)
+        [total - idle, total]
+      end
+
+      def cpu_shares(before, after)
+        return {} if before.nil? || after.nil?
+
+        total = cpu_total(after) - cpu_total(before)
+        return {} if total <= 0
+
+        out = {}
+        CPU_SHARES.each do |name, columns|
+          next if columns.any? { |i| i >= before.size || i >= after.size }
+
+          moved = columns.sum { |i| after[i] - before[i] }
+          return {} if moved.negative?
+
+          out[name] = [100.0, 100.0 * moved / total].min.round(1)
+        end
+        out
+      end
+
+      def disk_counters(text)
+        read = written = 0
+        found = false
+        text.each_line do |line|
+          fields = line.split
+          next if fields.size < 10
+
+          name = fields[2]
+          next if name.match?(IO_NOT_A_DISK) || name.match?(IO_PARTITION)
+          next unless fields[5].match?(/\A\d+\z/) && fields[9].match?(/\A\d+\z/)
+
+          read += fields[5].to_i * IO_SECTOR
+          written += fields[9].to_i * IO_SECTOR
+          found = true
+        end
+        found ? [read, written] : nil
+      end
+
+      def net_counters(text)
+        rx = tx = 0
+        found = false
+        text.each_line do |line|
+          next unless line.include?(":")
+
+          name, rest = line.split(":", 2)
+          name = name.strip
+          next if name.empty? || name.match?(NET_NOT_REAL)
+
+          fields = rest.split
+          next if fields.size < 9 || !fields[0].match?(/\A\d+\z/) || !fields[8].match?(/\A\d+\z/)
+
+          rx += fields[0].to_i
+          tx += fields[8].to_i
+          found = true
+        end
+        found ? [rx, tx] : nil
+      end
+
+      def rate_mbs(before, after, seconds)
+        return nil if before.nil? || after.nil?
+        return nil if seconds <= 0 || seconds > IO_MAX_PAIR_SECONDS
+
+        moved = after - before
+        return nil if moved.negative?
+
+        (moved.to_f / seconds / MB).round(1)
+      end
+
+      def cpu_percent(before, after)
+        return nil if before.nil? || after.nil?
+
+        busy = after[0] - before[0]
+        total = after[1] - before[1]
+        return nil if total <= 0 || busy.negative?
+
+        [100.0, 100.0 * busy / total].min.round(1)
+      end
+
+      # Used is total - MemAvailable, never total - MemFree: page cache is not pressure.
+      def memory_mb(text)
+        values = {}
+        text.each_line do |line|
+          name, _, rest = line.partition(":")
+          first = rest.split.first
+          values[name] = first.to_i / 1024 if first&.match?(/\A\d+\z/)
+        end
+        out = {}
+        if values.key?("MemTotal")
+          out["mem_total_mb"] = values["MemTotal"]
+          out["mem_used_mb"] = [0, values["MemTotal"] - values["MemAvailable"]].max if values.key?("MemAvailable")
+        end
+        if values.key?("SwapTotal")
+          out["swap_total_mb"] = values["SwapTotal"]
+          out["swap_used_mb"] = [0, values["SwapTotal"] - values["SwapFree"]].max if values.key?("SwapFree")
+        end
+        out
+      end
+
+      # The real disks behind /proc/mounts, one per device under its shortest
+      # mount point, so a bind mount isn't reported as a second disk; a mount
+      # that isn't a directory (Docker bind-mounts /etc/hosts) doesn't win.
+      # Off Linux, the one filesystem we can name without guessing: "/".
+      def mount_points(mounts_text, is_dir = nil)
+        return ["/"] if mounts_text.to_s.empty?
+
+        is_dir ||= ->(path) { File.directory?(path) }
+        dirs = {}
+        any = {}
+        mounts_text.each_line do |line|
+          fields = line.split
+          next if fields.size < 3
+
+          device = fields[0]
+          mount = fields[1].gsub("\\040", " ")
+          next unless FSTYPES.include?(fields[2]) && mount.start_with?("/")
+
+          any[device] = mount if !any.key?(device) || mount.size < any[device].size
+          confirmed = begin
+            is_dir.call(mount)
+          rescue StandardError
+            false
+          end
+          dirs[device] = mount if confirmed && (!dirs.key?(device) || mount.size < dirs[device].size)
+        end
+        any.map { |device, mount| dirs[device] || mount }.uniq.sort
+      end
+
+      # df's POSIX output: "<fs> <total> <used> <available> <capacity> <mount>",
+      # in 1024-byte blocks (-k) or inodes (-i), by mount point.
+      def parse_df(text)
+        out = {}
+        text.to_s.each_line.drop(1).each do |line|
+          m = /\A\S+\s+(\d+)\s+(\d+)\s+(\d+)\s+\S+\s+(.+?)\s*\z/.match(line)
+          out[m[4]] = [m[1].to_i, m[2].to_i, m[3].to_i] if m
+        end
+        out
+      end
+
+      def run(argv, timeout = DF_TIMEOUT)
+        reader, writer = IO.pipe
+        pid = Process.spawn(*argv, out: writer, err: File::NULL, in: File::NULL)
+        writer.close
+        output = +""
+        deadline = SiteGoose.clock + timeout
+        loop do
+          left = deadline - SiteGoose.clock
+          break if left <= 0 || !reader.wait_readable(left)
+
+          chunk = reader.read_nonblock(65_536, exception: false)
+          break if chunk.nil?
+
+          output << chunk unless chunk == :wait_readable
+        end
+        begin
+          Process.kill("KILL", pid) if Process.waitpid(pid, Process::WNOHANG).nil?
+        rescue SystemCallError
+          nil
+        end
+        begin
+          Process.waitpid(pid)
+        rescue SystemCallError
+          nil
+        end
+        output
+      rescue StandardError
+        ""
+      ensure
+        reader&.close unless reader&.closed?
+        writer&.close unless writer&.closed?
+      end
+
+      def df(points)
+        return [{}, {}] if points.empty?
+
+        [parse_df(run(["df", "-kP", *points])), parse_df(run(["df", "-iP", *points]))]
+      end
+
+      # Both figures are df's own, so a customer can check them. One
+      # filesystem behind two devices (a container's overlay and the host disk
+      # bind-mounted into it) reports the same sizes, so a repeat is dropped.
+      def disk_rows(points, blocks = nil, inodes = nil)
+        blocks, inodes = df(points) if blocks.nil?
+        seen = {}
+        rows = []
+        points.each do |mount|
+          figures = blocks[mount]
+          next unless figures
+
+          _, used_kb, avail_kb = figures
+          used = used_kb * 1024
+          total = used + avail_kb * 1024
+          next if total.zero?
+
+          row = { "mount" => mount, "total_mb" => total / MB, "used_mb" => used / MB }
+          next if row["total_mb"] < MIN_DISK_MB
+          next if seen[[total, used]]
+
+          seen[[total, used]] = true
+          # A filesystem can refuse a write with gigabytes free once these meet.
+          files = inodes && inodes[mount]
+          if files && files[0].positive?
+            row["inodes_total"] = files[0]
+            row["inodes_used"] = files[1]
+          end
+          rows << row
+          break if rows.size >= MAX_DISKS
+        end
+        rows
+      end
+
+      def host_uptime(text)
+        first = text.to_s.split.first
+        first&.match?(/\A\d+(\.\d+)?\z/) ? first.to_f.to_i : nil
+      end
+
+      def load_average(text)
+        parts = text.to_s.split
+        return {} if parts.size < 3 || !parts.first(3).all? { |p| p.match?(/\A\d+(\.\d+)?\z/) }
+
+        { "load1" => parts[0].to_f.round(2), "load5" => parts[1].to_f.round(2), "load15" => parts[2].to_f.round(2) }
+      end
+
+      # This package runs inside the application, so unlike the agent it can
+      # honestly report the application's own memory.
+      def process_rss_mb(text)
+        text.each_line do |line|
+          next unless line.start_with?("VmRSS:")
+
+          kb = line.split[1]
+          return kb&.match?(/\A\d+\z/) ? [1, kb.to_i / 1024].max : nil
+        end
+        nil
+      end
+    end
+
+    attr_reader :rows, :every, :enabled
+
+    # Options: server_name (the server's name as the dashboard shows it).
+    def initialize(core, server_name: nil, **_)
+      @core = core
+      @server_name = SiteGoose.text(server_name.nil? ? SiteGoose.hostname : server_name, 100)
+      # minute (ISO) -> row, so a minute sampled twice is kept once.
+      @rows = {}
+      @lock = Mutex.new
+      @every = EVERY
+      @enabled = true
+      @last_cpu = Resources.cpu_fields(Resources.read_text("/proc/stat"))
+      @last_disk = Resources.disk_counters(Resources.read_text("/proc/diskstats"))
+      @last_net = Resources.net_counters(Resources.read_text("/proc/net/dev"))
+      @last_at = Time.now.to_f
+      @sampler = core.valid? ? Periodic.new(SAMPLE_EVERY) { sample; SAMPLE_EVERY } : nil
+      @job = core.valid? ? Periodic.new(EVERY) { report } : nil
+      Forking.register(self)
+    end
+
+    def after_fork_in_child
+      @lock = Mutex.new
+      @rows = {}
+    end
+
+    # Takes one sample, covering the time since the last one.
+    def sample(at = nil)
+      at ||= Time.now.to_f
+      cpu = Resources.cpu_fields(Resources.read_text("/proc/stat"))
+      disk = Resources.disk_counters(Resources.read_text("/proc/diskstats"))
+      net = Resources.net_counters(Resources.read_text("/proc/net/dev"))
+      percent = Resources.cpu_percent(@last_cpu && Resources.cpu_times_of(@last_cpu), cpu && Resources.cpu_times_of(cpu))
+      elapsed = at - @last_at
+      # The minute the interval started in, not the one it ended in.
+      row = { "minute" => Time.at((@last_at.to_i / 60) * 60).utc.iso8601(3) }
+      shares = Resources.cpu_shares(@last_cpu, cpu)
+      rates = {}
+      if @last_disk && disk
+        rates["disk_read_mbs"] = Resources.rate_mbs(@last_disk[0], disk[0], elapsed)
+        rates["disk_write_mbs"] = Resources.rate_mbs(@last_disk[1], disk[1], elapsed)
+      end
+      if @last_net && net
+        rates["net_rx_mbs"] = Resources.rate_mbs(@last_net[0], net[0], elapsed)
+        rates["net_tx_mbs"] = Resources.rate_mbs(@last_net[1], net[1], elapsed)
+      end
+      @last_cpu = cpu
+      @last_disk = disk
+      @last_net = net
+      @last_at = at
+      row.merge!(shares)
+      rates.each { |name, value| row[name] = value unless value.nil? }
+      row["cpu_pct"] = percent if percent && elapsed >= MIN_CPU_SECONDS
+      cores = begin
+        Etc.nprocessors
+      rescue StandardError
+        nil
+      end
+      row["cpu_cores"] = cores if cores&.positive?
+      row.merge!(Resources.load_average(Resources.read_text("/proc/loadavg")))
+      row.merge!(Resources.memory_mb(Resources.read_text("/proc/meminfo")))
+      disks = Resources.disk_rows(Resources.mount_points(Resources.read_text("/proc/mounts")))
+      row["disks"] = disks unless disks.empty?
+      rss = Resources.process_rss_mb(Resources.read_text("/proc/self/status"))
+      row["proc_rss_mb"] = rss if rss
+      uptime = Resources.host_uptime(Resources.read_text("/proc/uptime"))
+      row["host_uptime_s"] = uptime if uptime
+      @lock.synchronize do
+        # Off in the dashboard: only the newest minute is kept, as the one row
+        # that asks the collector, now and then, whether it is back on.
+        @rows.clear unless @enabled
+        @rows[row["minute"]] = row
+        # Oldest first out: a long outage keeps the recent minutes.
+        @rows.delete(@rows.keys.min) while @rows.size > MAX_ROWS
+      end
+      row
+    end
+
+    # Sends what's buffered. Rows are kept only when the collector couldn't
+    # take them and retrying may work.
+    def flush
+      minutes, rows = @lock.synchronize do
+        keys = @rows.keys.sort
+        [keys, keys.map { |m| @rows[m] }]
+      end
+      return true if rows.empty?
+
+      body = { "site_id" => @core.site_id, "server_name" => @server_name, "runtime" => RUNTIME[0, 32], "rows" => rows }
+      res = @core.request("resources", "/events/resources", auth: :bearer, body: body)
+      return false if res.nil?
+
+      unless res.ok?
+        # Unreachable or storage down: these rows are still good. A 4xx means
+        # resending won't make them acceptable.
+        return false if res.status >= 500
+
+        drop(minutes)
+        return true
+      end
+      answer = res.json
+      if answer.is_a?(Hash)
+        @every = [RETRY_EVERY, answer["every"].to_i].max if answer["every"].is_a?(Numeric) && answer["every"].positive?
+        @enabled = answer["enabled"] != false
+      end
+      drop(minutes)
+      true
+    end
+
+    def drop(minutes)
+      @lock.synchronize { minutes.each { |m| @rows.delete(m) } }
+    end
+
+    def report
+      unless @core.on?("resources")
+        # Switched off: keep nothing, and ask again later.
+        @lock.synchronize { @rows.clear }
+        return [OFF_EVERY, @every].max
+      end
+      sent = flush
+      # The collector said the module is off: ask again later, as the agent does.
+      return [OFF_EVERY, @every].max unless @enabled
+
+      sent ? @every : RETRY_EVERY
+    end
+
+    def close
+      @sampler&.stop
+      @job&.stop
+      Forking.unregister(self)
+      # What's buffered goes out, as with every other module.
+      flush if @core.on?("resources") && @enabled
+    end
+  end
+end

data/lib/sitegoose/sidekiq.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+# Sidekiq: every job runs as a platform.job() named after its class (the
+# ActiveJob class when Sidekiq runs ActiveJob). Its errors carry
+# "/(job) <class>" as their route, and each run is timed (Health -> Server
+# speed -> Jobs).
+#
+#     # config/initializers/sidekiq.rb
+#     require "sitegoose/sidekiq"
+#     SiteGoose::Sidekiq.install
+#
+# A failure that will be retried is timed as failed but not sent: the error
+# is sent when the last attempt fails (retry: false, retry: 0, or the retries
+# used up), or when Sidekiq gives the job up (its death handlers: a job sent
+# to the Dead set, sidekiq_retry_in returning :kill). Sidekiq stopping a job
+# mid-run (Sidekiq::Shutdown) is neither timed nor sent: it runs again. What's
+# waiting is sent when Sidekiq shuts down.
+
+module SiteGoose
+  module Sidekiq
+    SKIP = :@__sitegoose_skip
+    DEFAULT_RETRIES = 25
+
+    module_function
+
+    # Adds the server middleware, the death handler and the shutdown flush.
+    # platform: default the configured one (SiteGoose.platform).
+    def install(platform = nil)
+      ::Sidekiq.configure_server do |config|
+        config.server_middleware { |chain| chain.add ServerMiddleware, platform }
+        config.death_handlers << ->(job, error) { Sidekiq.died(platform, job, error) } if config.respond_to?(:death_handlers)
+        config.on(:shutdown) { SiteGoose.within { (platform || SiteGoose.platform)&.flush } } if config.respond_to?(:on)
+      end
+    end
+
+    def name_of(job)
+      (job["wrapped"] || job["class"]).to_s
+    end
+
+    def max_retries
+      config = if ::Sidekiq.respond_to?(:default_configuration) then ::Sidekiq.default_configuration
+               elsif ::Sidekiq.respond_to?(:options) then ::Sidekiq.options
+               end
+      value = config && config[:max_retries]
+      value.is_a?(Integer) ? value : DEFAULT_RETRIES
+    rescue StandardError
+      DEFAULT_RETRIES
+    end
+
+    # Whether this run is the job's last: no retries, or the retries used up.
+    # retry_count is nil on the first run and 0 on the first retry.
+    def last_attempt?(job)
+      retry_option = job["retry"]
+      return true if retry_option == false || retry_option.nil?
+
+      allowed = retry_option == true ? max_retries : retry_option.to_i
+      attempts = job["retry_count"].nil? ? 0 : job["retry_count"].to_i + 1
+      attempts >= allowed
+    end
+
+    def died(platform, job, error)
+      made = platform || SiteGoose.platform
+      return unless made && error.is_a?(Exception)
+
+      error.remove_instance_variable(SKIP) if error.instance_variable_defined?(SKIP)
+      route = "/(job) #{SiteGoose.text(name_of(job), 180)}"
+      made.capture_exception(error, status: 500, kind: "error",
+                                    request: { "method" => "JOB", "path" => route, "route" => route, "session_id" => "" })
+    rescue StandardError
+      nil
+    end
+
+    class ServerMiddleware
+      include ::Sidekiq::ServerMiddleware if defined?(::Sidekiq::ServerMiddleware)
+
+      def initialize(platform = nil)
+        @platform = platform
+      end
+
+      def call(_worker, job, _queue, &block)
+        made = @platform || SiteGoose.platform
+        return yield if made.nil?
+
+        Forking.check
+        last = Sidekiq.last_attempt?(job)
+        made.job(Sidekiq.name_of(job), report: lambda { |error|
+          # Retried: Rails' own error report (its executor wraps the job) is
+          # told to leave it too.
+          error.instance_variable_set(SKIP, true) if !last && !error.frozen?
+          last
+        }, &block)
+      end
+    end
+  end
+end