sitegoose 0.3.3

data/lib/sitegoose/firewall.rb

@@ -0,0 +1,417 @@
+# frozen_string_literal: true
+
+# Firewall module: rules from the collector (GET /events/firewall/rules,
+# protocol in sdk/protocols/firewall.md) every minute or so; every request is
+# checked against them in memory, so the app never waits on the collector.
+# What was blocked (or, in monitor mode, would have been) goes back in batches.
+#
+# Fails open: until rules arrive, and whenever the collector can't be reached,
+# requests pass (with the last rules that arrived, if any). The rate limit
+# counts in this process: with several workers, pass `counter` (e.g. Redis)
+# for one count across them.
+
+require "json"
+
+module SiteGoose
+  class Firewall
+    PROTOCOL = 1
+    FLUSH_EVERY = 5
+    BATCH = 200
+    QUEUE_MAX = 2000
+    COUNTER_MAX = 100_000
+    # The layers this server runs (X-Firewall-Layers): the address firewall and the browser check.
+    LAYERS = "app check"
+    # Files a page load asks for by the dozen: not counted against `rate`
+    # (the agent's nginx limit leaves the same extensions out).
+    STATIC = /\.(?:css|js|mjs|map|png|jpe?g|gif|webp|avif|svg|ico|bmp|woff2?|ttf|otf|eot|mp4|webm|ogg|mp3|wav|pdf|wasm)\z/i.freeze
+    STATIC_PREFIXES = %w[/assets/ /packs/ /vite/ /static/ /_next/static/].freeze
+
+    attr_reader :pass_key, :queue, :bans
+    attr_accessor :rules, :etag
+
+    # The rules' probes (addresses of files no visitor asks for: .env,
+    # .git/...) -> {exact:, prefix:, ban:}, or nil.
+    def self.compile_probes(probes)
+      return nil unless probes.is_a?(Hash)
+
+      paths = ->(key) { Array(probes[key]).select { |p| p.is_a?(String) && !p.empty? }.map(&:downcase) }
+      exact = paths.call("exact").to_h { |p| [p, true] }
+      prefix = paths.call("prefix")
+      return nil if exact.empty? && prefix.empty?
+
+      ban = probes["ban"].is_a?(Numeric) ? [0, probes["ban"].to_i].max : 0
+      { exact: exact, prefix: prefix, ban: ban }
+    end
+
+    # Read from the path the router sees, so "/admin;x.css" isn't one.
+    def self.static?(target)
+      path = RequestRules.normalize_target(target)
+      STATIC_PREFIXES.any? { |prefix| path.start_with?(prefix) } || path.match?(STATIC)
+    end
+
+    # Who the rate limit, the login count and the bans here are about: an
+    # IPv4 address (::ffff:a.b.c.d as a.b.c.d), or an IPv6 address's /64,
+    # which one machine usually holds whole -- as the browser check binds.
+    #
+    # Written the same way by every package, so a site whose servers run
+    # different languages on one Redis counts a visitor once: "198.51.100.7",
+    # or the /64 as its network, RFC 5952 (lower-case, compressed):
+    # "2001:db8:1:2::/64". The browser check's pass binds its own way
+    # (Challenge.who), which this doesn't change.
+    def self.who(ip)
+      key = IP.key(ip)
+      return ip.to_s if key.nil?
+      return IPAddr.new(key & 0xFFFF_FFFF, Socket::AF_INET).to_s if (key >> 32) == 0xFFFF
+
+      "#{IPAddr.new((key >> 64) << 64, Socket::AF_INET6)}/64"
+    end
+
+    # True when the path (no query, lower-cased, "//" as "/") is a probe, as
+    # sent or as the router reads it (";params" dropped, "." and ".."
+    # resolved, decoded once: "/x/..;/.env" is "/.env").
+    def self.probe?(path, probes)
+      return false unless probes
+
+      clean = SiteGoose.text(path).split(/[?#]/, 2)[0].to_s.downcase.gsub(%r{/{2,}}, "/")
+      [clean, RequestRules.normalize_target(path)].uniq.any? do |text|
+        probes[:exact].key?(text) || probes[:prefix].any? { |p| text.start_with?(p) }
+      end
+    end
+
+    # The rules' browser check -> {skip:, exempt:}, or nil when the site isn't to be checked.
+    def self.compile_challenge(data)
+      return nil unless data.is_a?(Hash)
+
+      { skip: Challenge.compile_skip(data["skip"]), exempt: IpSet.new(Array(data["exempt"])) }
+    end
+
+    # This machine's host name for X-Server-Name: plain ASCII, 100 characters at most.
+    def self.server_name
+      SiteGoose.hostname.gsub(/[^\x21-\x7e]/, "")[0, 100]
+    end
+
+    def self.int(value, fallback = 0)
+      value.is_a?(Numeric) ? value.to_i : (value.is_a?(String) && value.match?(/\A\d+\z/) ? value.to_i : fallback)
+    end
+
+    # Options: trust_proxy, mode ("monitor"/"block": overrides the dashboard),
+    # counter (callable(key, ttl) -> hits, shared between workers), cache_file
+    # (rules kept there between restarts).
+    def initialize(core, trust_proxy: nil, mode: nil, counter: nil, cache_file: nil, **_)
+      @core = core
+      @trust_proxy = trust_proxy
+      @mode = %w[monitor block].include?(mode.to_s) ? mode.to_s : nil
+      @counter = counter
+      @cache_file = cache_file
+      @rules = nil
+      @etag = ""
+      @lock = Mutex.new
+      @counts = {}
+      @queue = {}
+      # Bot protection here: address -> when its ban ends; address -> [window start, failed logins].
+      @bans = {}
+      @logins = {}
+      # The browser check's passes are signed with the site secret, so any of
+      # the site's servers accepts one another handed out.
+      @pass_key = Challenge.pass_key(core.secret)
+      load_cache
+      @jobs = []
+      return unless core.valid?
+
+      @jobs = [Periodic.new { sync }, Periodic.new(FLUSH_EVERY) { flush; FLUSH_EVERY }]
+      Forking.register(self)
+    end
+
+    def load_cache
+      return unless @cache_file && File.exist?(@cache_file)
+
+      cached = JSON.parse(File.read(@cache_file))
+      @rules = compile(cached["rules"])
+      @etag = cached["etag"].to_s
+    rescue StandardError
+      nil
+    end
+
+    def after_fork_in_child
+      @lock = Mutex.new
+      @queue = {}
+    end
+
+    def compile(data)
+      raise ArgumentError, "unknown rules version" unless data.is_a?(Hash) && data["v"] == PROTOCOL
+
+      rate = data["rate"].is_a?(Hash) ? data["rate"] : nil
+      rate = if rate && Firewall.int(rate["limit"]).positive?
+               { limit: Firewall.int(rate["limit"]), window: [1, Firewall.int(rate["window"], 60).nonzero? || 60].max }
+             end
+      login = data["login"].is_a?(Hash) && Firewall.int(data["login"]["limit"]).positive? ? data["login"] : nil
+      {
+        enabled: data["enabled"] == true,
+        mode: data["mode"] == "block" ? "block" : "monitor",
+        sync: [MIN_SYNC, Firewall.int(data["sync"], RETRY).nonzero? || RETRY].max,
+        allow: IpSet.new(Array(data["allow"])),
+        # Crawlers the rate limit must not catch (absent in older rules).
+        goodbots: IpSet.new(Array(data["goodbots"])),
+        deny: IpSet.new(Array(data["deny"])),
+        list: IpSet.new(Array(data["lists"])),
+        shared: IpSet.new(Array(data["shared"])),
+        rate: rate,
+        banned: IpSet.new(Array(data["banned"])),
+        ban: data["ban"].is_a?(Hash) ? Firewall.int(data["ban"]["seconds"]) : 0,
+        login: login,
+        probes: Firewall.compile_probes(data["probes"]),
+        # Virtual patching: rules that match the request, not the address.
+        requests: RequestRules.compile(data["requests"]),
+        # The browser check: present only when the site is to be checked.
+        challenge: Firewall.compile_challenge(data["challenge"])
+      }
+    end
+
+    def sync
+      headers = { "X-Firewall-Layers" => LAYERS }
+      name = Firewall.server_name
+      headers["X-Server-Name"] = name unless name.empty?
+      headers["If-None-Match"] = @etag unless @etag.empty?
+      res = @core.request("firewall", "/events/firewall/rules?site_id=#{@core.site_id}", method: "GET", auth: :bearer, headers: headers)
+      if res && res.status == 200
+        data = res.json
+        begin
+          @rules = compile(data)
+          @etag = res.headers["etag"].to_s
+          write_cache(data) if @cache_file
+        rescue StandardError => e
+          @core.log("firewall", "rules not used:", e.message)
+        end
+      end
+      @rules ? @rules[:sync] : RETRY
+    end
+
+    # Written beside the file and renamed over it, so a crash or a second
+    # worker writing at once never leaves half a file to load.
+    def write_cache(data)
+      temp = "#{@cache_file}.#{Process.pid}.#{Thread.current.object_id}.tmp"
+      File.write(temp, JSON.generate({ "etag" => @etag, "rules" => data }))
+      File.rename(temp, @cache_file)
+    rescue StandardError => e
+      File.delete(temp) rescue nil # rubocop:disable Style/RescueModifier
+      @core.log("firewall", "rules not kept in the cache file:", e.message)
+    end
+
+    def over_rate?(ip, rate)
+      who = Firewall.who(ip)
+      window = (Time.now.to_f / rate[:window]).floor
+      if @counter
+        return @counter.call("afw:#{@core.site_id}:#{who}:#{window}", rate[:window] * 2).to_i > rate[:limit]
+      end
+
+      @lock.synchronize do
+        entry = @counts[who]
+        if entry.nil? || entry[0] != window
+          # Taken out and put back, so the hash runs oldest window first and
+          # a full table loses the addresses that went quiet longest ago.
+          @counts.delete(who)
+          Firewall.make_room(@counts)
+          entry = [window, 0]
+          @counts[who] = entry
+        end
+        entry[1] += 1
+        entry[1] > rate[:limit]
+      end
+    end
+
+    # A table of addresses at COUNTER_MAX loses its oldest entries (insertion
+    # order), not everyone's count at once.
+    def self.make_room(table)
+      table.shift while table.size >= COUNTER_MAX
+    end
+
+    def banned_here?(ip)
+      who = Firewall.who(ip)
+      ends = @bans[who]
+      return false unless ends
+      return true if ends > Time.now.to_f
+
+      @lock.synchronize { @bans.delete(who) }
+      false
+    end
+
+    def ban_here(ip, seconds)
+      return unless seconds.positive?
+
+      who = Firewall.who(ip)
+      @lock.synchronize do
+        ends = [@bans.delete(who) || 0, Time.now.to_f + seconds].max
+        Firewall.make_room(@bans)
+        @bans[who] = ends
+      end
+    end
+
+    # A failed login on your site: counted here (past the site's limit the
+    # address is banned straight away) and reported, so it counts on the
+    # Security page and gets banned on the other servers too.
+    def login_failed(ip)
+      key = IP.key(ip)
+      rules = @rules
+      return if key.nil? || (rules && rules[:allow].has?(key))
+
+      login = rules && rules[:login]
+      if login
+        now = Time.now.to_f
+        window = Firewall.int(login["window"], 3600).nonzero? || 3600
+        who = Firewall.who(ip)
+        reached = @lock.synchronize do
+          entry = @logins[who]
+          if entry.nil? || now - entry[0] >= window
+            @logins.delete(who)
+            Firewall.make_room(@logins)
+            entry = @logins[who] = [now, 0]
+          end
+          entry[1] += 1
+          hit = entry[1] >= Firewall.int(login["limit"])
+          @logins.delete(who) if hit
+          hit
+        end
+        ban_here(ip.to_s, Firewall.int(login["ban"], 3600).nonzero? || 3600) if reached
+      end
+      report({ "ip" => ip.to_s, "rule" => "login", "action" => "monitor", "method" => "", "path" => "", "user_agent" => "" })
+    end
+
+    def report(event)
+      key = event.values_at("ip", "rule", "action", "method", "path").join("|")
+      @lock.synchronize do
+        row = @queue[key]
+        if row
+          row["hits"] += 1
+        elsif @queue.size < QUEUE_MAX
+          @queue[key] = event.merge("at" => SiteGoose.iso_now, "hits" => 1)
+        end
+      end
+    end
+
+    def flush
+      rows = @lock.synchronize do
+        taken = @queue.values
+        @queue = {}
+        taken
+      end
+      rows.each_slice(BATCH) do |batch|
+        @core.request("firewall", "/events/firewall/events", auth: :bearer, body: { "site_id" => @core.site_id, "events" => batch })
+      end
+    end
+
+    # -> {action: "allow" | "block" | "monitor", rule:, status:}. Allowed
+    # addresses skip every other rule. accept and cookie (the request's
+    # headers) are for the browser check: rule "challenge", status 503,
+    # answered with challenge_page().
+    def check(ip, method = "", path = "", user_agent = "", accept = "", cookie = "", now = nil)
+      allow = { action: "allow", rule: "", status: 200 }
+      rules = @rules
+      return allow if rules.nil? || !rules[:enabled]
+
+      key = IP.key(ip)
+      return allow if key.nil? || rules[:allow].has?(key)
+
+      ip = ip.to_s
+      action = (@mode || rules[:mode]) == "block" ? "block" : "monitor"
+      rule = address_rule(rules, key, ip)
+      if rule.empty? && Firewall.probe?(path, rules[:probes])
+        rule = "probe"
+        # Asking for a secret file: banned here in block mode; the report bans it on the other servers.
+        ban_here(ip, rules[:probes][:ban]) if action == "block"
+      end
+      if rule.empty?
+        matched = RequestRules.match(rules[:requests], method, path, user_agent)
+        if matched
+          rule = "request:#{matched['id']}"
+          # A rule the dashboard only watches passes even in block mode; one
+          # that blocks also bans the address here, as a flood does.
+          action = action == "block" && matched["action"] == "block" ? "block" : "monitor"
+          ban_here(ip, matched["ban"]) if action == "block" && matched["ban"].positive?
+        end
+      end
+      if rule.empty? && rules[:rate] && !rules[:goodbots].has?(key) && !Firewall.static?(path) && over_rate?(ip, rules[:rate])
+        rule = "rate"
+        # Banned here straight away in block mode, as a probe is; the report bans it on the other servers.
+        ban_here(ip, rules[:ban]) if action == "block"
+      end
+      event = lambda do |name, act|
+        { "ip" => ip, "rule" => name, "action" => act, "method" => SiteGoose.text(method, 10),
+          "path" => SiteGoose.text(path).split("?", 2)[0].to_s[0, 300], "user_agent" => SiteGoose.text(user_agent, 300) }
+      end
+      # Not a mode matter: the collector only sends `challenge` when the site
+      # is to be checked, and a match only watched (monitor) gets it too
+      # (firewall.md, step 7).
+      if (rule.empty? || action != "block") && needs_check?(rules, key, ip, method, path, accept, cookie, now)
+        report(event.call(rule, action)) unless rule.empty?
+        rule = "challenge"
+        action = "block"
+      end
+      return allow if rule.empty?
+
+      report(event.call(rule, action))
+      { action: action, rule: rule, status: rule == "rate" ? 429 : (rule == "challenge" ? 503 : 403) }
+    end
+
+    # The rules about the address alone (deny, bans, lists, shared): "" when none.
+    def address_rule(rules, key, ip)
+      if rules[:deny].has?(key) then "deny"
+      elsif rules[:banned].has?(key) || banned_here?(ip) then "ban"
+      elsif rules[:list].has?(key) then "list"
+      elsif rules[:shared].has?(key) then "shared"
+      else ""
+      end
+    end
+
+    # The address rules only, for the tracker's own addresses (the proxy):
+    # a denied or banned address gets nothing from us either, but beacons
+    # are never rate-limited, checked or matched against request rules.
+    # -> the same shape as check().
+    def check_address(ip, method = "", path = "", user_agent = "")
+      allow = { action: "allow", rule: "", status: 200 }
+      rules = @rules
+      return allow if rules.nil? || !rules[:enabled]
+
+      key = IP.key(ip)
+      return allow if key.nil? || rules[:allow].has?(key)
+
+      rule = address_rule(rules, key, ip.to_s)
+      return allow if rule.empty?
+
+      action = (@mode || rules[:mode]) == "block" ? "block" : "monitor"
+      report({ "ip" => ip.to_s, "rule" => rule, "action" => action, "method" => SiteGoose.text(method, 10),
+               "path" => SiteGoose.text(path).split("?", 2)[0].to_s[0, 300], "user_agent" => SiteGoose.text(user_agent, 300) })
+      { action: action, rule: rule, status: 403 }
+    end
+
+    # Step 7: the browser check is on, the request asks for a page outside
+    # skip, the address is in none of allow (already passed), goodbots and
+    # exempt, and it has no valid pass.
+    def needs_check?(rules, key, ip, method, path, accept, cookie, now)
+      check = rules[:challenge]
+      return false if check.nil? || !Challenge.asks_for_page?(method, path, accept, check[:skip])
+      return false if rules[:goodbots].has?(key) || check[:exempt].has?(key)
+
+      !Challenge.pass?(@pass_key, ip, cookie, now)
+    end
+
+    # [503, headers, body] for a request check() gave rule "challenge".
+    def challenge_page(ip, method)
+      Challenge.page(@pass_key, ip, method)
+    end
+
+    def retry_after
+      rules = @rules
+      rules && rules[:rate] ? rules[:rate][:window].to_s : "60"
+    end
+
+    def client_ip(remote, headers)
+      SiteGoose.client_ip(remote, headers, @trust_proxy)
+    end
+
+    def close
+      @jobs.each(&:stop)
+      Forking.unregister(self)
+      flush
+    end
+  end
+end

data/lib/sitegoose/inventory.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+# Software inventory module (protocol in sdk/protocols/inventory.md): the
+# gems the app runs and its Ruby version, sent to POST /events/inventory a
+# minute after start and then once a day (the collector's `every`), so the
+# Security check can match them against known security holes (OSV's
+# "RubyGems" ecosystem). Runs in a background thread, never in a request;
+# names and versions only.
+#
+# Where the list comes from: the gems loaded in this process (Bundler's, so
+# what Gemfile.lock pins and the groups the app runs with), each marked
+# `direct` when the Gemfile asks for it by name. With no Bundler (a plain
+# `ruby app.rb`), Gemfile.lock next to the app is read instead.
+
+module SiteGoose
+  class Inventory
+    FIRST = 60
+    RETRY_EVERY = 60 * 60
+    EVERY = 24 * 60 * 60
+    MAX_PACKAGES = 5000
+    ECOSYSTEM = "RubyGems"
+
+    # "    rack (3.1.8)" and "    nokogiri (1.16.7-x86_64-linux)" under a
+    # GEM, GIT or PATH section's specs, as [name, version].
+    def self.lockfile_specs(text)
+      found = {}
+      in_specs = false
+      text.to_s.each_line do |line|
+        if line.match?(/\A\S/)
+          in_specs = false
+        elsif line.match?(/\A  specs:\s*\z/)
+          in_specs = true
+        elsif in_specs && (m = /\A {4}([^\s(]+) \(([^)]+)\)\s*\z/.match(line))
+          version = m[2].split("-", 2)[0]
+          found[m[1]] ||= version
+        end
+      end
+      found
+    end
+
+    # The Gemfile's own dependencies, by name, from a Gemfile.lock's DEPENDENCIES.
+    def self.lockfile_direct(text)
+      names = {}
+      in_deps = false
+      text.to_s.each_line do |line|
+        if line.match?(/\A\S/)
+          in_deps = line.start_with?("DEPENDENCIES")
+        elsif in_deps && (m = /\A {2}([^\s!(]+)/.match(line))
+          names[m[1]] = true
+        end
+      end
+      names
+    end
+
+    def self.bundler_direct
+      return {} unless defined?(::Bundler) && ::Bundler.respond_to?(:definition)
+
+      ::Bundler.definition.dependencies.to_h { |dep| [dep.name, true] }
+    rescue StandardError, ScriptError
+      {}
+    end
+
+    # [{"ecosystem", "name", "version", "direct"?}], one per gem, sorted by name.
+    def self.installed_packages(root = nil)
+      specs = {}
+      direct = {}
+      if defined?(::Gem) && ::Gem.respond_to?(:loaded_specs)
+        ::Gem.loaded_specs.each_value do |spec|
+          name = spec.name.to_s
+          version = spec.version.to_s
+          specs[name] ||= version unless name.empty? || version.empty?
+        end
+        direct = bundler_direct
+      end
+      if specs.size <= 1
+        lock = File.join((root || Dir.pwd).to_s, "Gemfile.lock")
+        if File.file?(lock)
+          text = File.read(lock, encoding: "UTF-8")
+          specs = lockfile_specs(text).merge(specs)
+          direct = lockfile_direct(text) if direct.empty?
+        end
+      end
+      specs.sort_by { |name, _| name.downcase }.map do |name, version|
+        row = { "ecosystem" => ECOSYSTEM, "name" => name, "version" => version.delete_prefix("v") }
+        row["direct"] = true if direct[name]
+        row
+      end
+    rescue StandardError
+      []
+    end
+
+    # Options: host (the server's name; default the host name), root (where
+    # Gemfile.lock is when Bundler isn't loaded; default the working directory).
+    def initialize(core, host: nil, root: nil, **_)
+      @core = core
+      @root = root
+      @host = SiteGoose.text(host.nil? ? SiteGoose.hostname : host).gsub(/[^A-Za-z0-9._-]/, "-")[0, 100]
+      @host = "server" if @host.empty?
+      @job = core.valid? ? Periodic.new(FIRST) { report } : nil
+    end
+
+    def body
+      packages = Inventory.installed_packages(@root)
+      {
+        "site_id" => @core.site_id,
+        "host" => @host,
+        "at" => SiteGoose.iso_now,
+        "runtime" => { "ruby" => RUBY_VERSION },
+        "packages" => packages.first(MAX_PACKAGES),
+        "truncated" => packages.size > MAX_PACKAGES
+      }
+    end
+
+    def report
+      return RETRY_EVERY unless @core.on?("inventory")
+
+      res = @core.request("inventory", "/events/inventory", auth: :bearer, body: body)
+      return RETRY_EVERY if res.nil? || !res.ok?
+
+      answer = res.json
+      every = answer.is_a?(Hash) ? answer["every"] : nil
+      every.is_a?(Integer) && every.positive? ? [RETRY_EVERY, every].max : EVERY
+    end
+
+    def flush; end
+
+    def close
+      @job&.stop
+    end
+  end
+end