sitegoose 0.3.3

data/lib/sitegoose/ip.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require "ipaddr"
+
+module SiteGoose
+  # Addresses as integers in IPv6 space, IPv4 kept as ::ffff:a.b.c.d, so one
+  # sorted list holds both (sdk/protocols/firewall.md: treat IPv4-mapped IPv6
+  # as IPv4).
+  module IP
+    MAPPED = 0xFFFF << 32
+    V4 = /\A(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\z/.freeze
+
+    module_function
+
+    # An address as an Integer in IPv6 space, or nil when it isn't one.
+    def key(text)
+      s = text.to_s.strip
+      s = s.split("%", 2)[0] if s.include?("%")
+      return nil if s.empty? || s.include?("/")
+
+      if (m = V4.match(s))
+        octets = m.captures
+        return nil if octets.any? { |o| (o.size > 1 && o.start_with?("0")) || o.to_i > 255 }
+
+        return MAPPED | octets.reduce(0) { |acc, o| (acc << 8) | o.to_i }
+      end
+      return nil unless s.include?(":")
+
+      IPAddr.new(s).to_i
+    rescue IPAddr::Error, ArgumentError
+      nil
+    end
+
+    # "1.2.3.0/24" -> [start, end] as key()s, or nil. nil too for an IPv6
+    # range that holds all of ::ffff:0:0/96 (::/64, ::/0): with IPv4 kept as
+    # ::ffff:a.b.c.d it would match every IPv4 visitor, a /64 that is one
+    # machine in IPv6 being the whole IPv4 internet here.
+    def range(entry)
+      s = entry.to_s.strip
+      return nil if s.empty?
+
+      address, bits = s.split("/", 2)
+      net = IPAddr.new(address.to_s.split("%", 2)[0])
+      if bits
+        return nil unless bits.match?(/\A\d{1,3}\z/)
+
+        prefix = bits.to_i
+        return nil if prefix > (net.ipv4? ? 32 : 128)
+
+        net = net.mask(prefix)
+      end
+      span = net.to_range
+      first = span.first.to_i
+      last = span.last.to_i
+      return [MAPPED | first, MAPPED | last] if net.ipv4?
+      return nil if first <= MAPPED && last >= (MAPPED | 0xFFFFFFFF)
+
+      [first, last]
+    rescue IPAddr::Error, ArgumentError
+      nil
+    end
+  end
+
+  # Addresses and ranges, merged and sorted, looked up by binary search.
+  class IpSet
+    def initialize(entries = [])
+      ranges = Array(entries).filter_map { |e| e.is_a?(String) ? IP.range(e) : nil }.sort
+      @starts = []
+      @ends = []
+      ranges.each do |first, last|
+        if !@ends.empty? && first <= @ends[-1] + 1
+          @ends[-1] = last if last > @ends[-1]
+        else
+          @starts << first
+          @ends << last
+        end
+      end
+    end
+
+    def has?(key)
+      return false if key.nil? || @starts.empty?
+
+      after = @starts.bsearch_index { |start| start > key } || @starts.size
+      i = after - 1
+      i >= 0 && @ends[i] >= key
+    end
+    alias include? has?
+
+    def size
+      @starts.size
+    end
+  end
+
+  # Cloudflare's published ranges (cloudflare.com/ips), the same list as
+  # sdk/protocols/cloudflare-ranges.json (test/firewall_test.rb compares them):
+  # CF-Connecting-IP is believed only from these, since anyone who reaches the
+  # origin directly can send it.
+  CLOUDFLARE = %w[
+    173.245.48.0/20 103.21.244.0/22 103.22.200.0/22 103.31.4.0/22 141.101.64.0/18
+    108.162.192.0/18 190.93.240.0/20 188.114.96.0/20 197.234.240.0/22 198.41.128.0/17
+    162.158.0.0/15 104.16.0.0/13 104.24.0.0/14 172.64.0.0/13 131.0.72.0/22
+    2400:cb00::/32 2606:4700::/32 2803:f800::/32 2405:b500::/32 2405:8100::/32
+    2a06:98c0::/29 2c0f:f248::/32
+  ].freeze
+
+  def self.from_cloudflare?(ip)
+    key = IP.key(ip)
+    return false if key.nil?
+
+    @cloudflare ||= IpSet.new(CLOUDFLARE)
+    @cloudflare.has?(key)
+  end
+
+  # The visitor's address. trust_proxy: nil/0 the connection's address; a
+  # number of proxies in X-Forwarded-For (true = 1); "cloudflare" reads
+  # CF-Connecting-IP, believed only when the connection comes from one of
+  # Cloudflare's addresses; a callable gets the headers and returns it.
+  #
+  # A hop that isn't an address (a name, "unknown", garbage a client sent)
+  # gives the connection's address instead, so the firewall still checks
+  # the request; a port on a hop ("1.2.3.4:5678", "[2001:db8::1]:443", as
+  # Azure's load balancer sends) is dropped.
+  def self.client_ip(remote, headers, trust_proxy)
+    remote = remote.to_s
+    if trust_proxy.respond_to?(:call)
+      given = hop(trust_proxy.call(headers))
+      return IP.key(given) ? given : remote
+    end
+
+    if trust_proxy.to_s == "cloudflare"
+      forwarded = hop(header(headers, "cf-connecting-ip"))
+      return from_cloudflare?(remote) && IP.key(forwarded) ? forwarded : remote
+    end
+    hops = if trust_proxy == true then 1
+           elsif trust_proxy.is_a?(Integer) then trust_proxy
+           elsif trust_proxy.is_a?(String) && trust_proxy.match?(/\A\d+\z/) then trust_proxy.to_i
+           else 0
+           end
+    if hops.positive?
+      chain = header(headers, "x-forwarded-for").split(",").map(&:strip).reject(&:empty?)
+      unless chain.empty?
+        chosen = hop(chain[[0, chain.size - hops].max])
+        return chosen if IP.key(chosen)
+      end
+    end
+    remote
+  end
+
+  # One hop as an address: "[v6]:port" and "v4:port" lose the port.
+  def self.hop(text)
+    s = text.to_s.strip
+    if s.start_with?("[")
+      close = s.index("]")
+      return close ? s[1...close] : s
+    end
+    found = /\A(\d{1,3}(?:\.\d{1,3}){3}):\d{1,5}\z/.match(s)
+    found ? found[1] : s
+  end
+end

data/lib/sitegoose/middleware.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module SiteGoose
+  # Rack middleware: the proxy, the firewall and the browser check,
+  # maintenance mode, then the app, timed, with its errors captured and the
+  # dashboard's security headers added. Put it first.
+  #
+  #     use SiteGoose::Middleware                      # configured elsewhere (Rails, configure, ENV)
+  #     use SiteGoose::Middleware, site_id: "...", secret: "...", endpoint: "..."
+  #     use SiteGoose::Middleware, platform            # one you made: SiteGoose::Platform.new(...)
+  #
+  # Never breaks a request: when the package itself fails, the request goes
+  # on to the app as if it weren't there.
+  class Middleware
+    def initialize(app, platform = nil, **options)
+      @app = app
+      @given = platform
+      SiteGoose.configure(**options) unless options.empty?
+    end
+
+    def platform
+      @given || SiteGoose.platform
+    rescue StandardError
+      nil
+    end
+
+    # Set on the env by the first copy: a second (the Railtie's and one added
+    # by hand, or an engine's) passes the request straight on, so nothing is
+    # checked, timed or reported twice.
+    SEEN = "sitegoose.seen"
+
+    def call(env)
+      return @app.call(env) if env[SEEN]
+
+      env[SEEN] = true
+      made = platform
+      return @app.call(env) if made.nil?
+
+      reached = false
+      made.rack_call(env) do
+        reached = true
+        @app.call(env)
+      end
+    rescue StandardError, ScriptError => e
+      # The app's own errors are raised on; ours never reach the visitor.
+      raise if reached
+
+      made&.core&.log("middleware", "failed, request passed:", e.message)
+      @app.call(env)
+    end
+  end
+end

data/lib/sitegoose/performance.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+# Performance & API monitoring module: times every request (from arriving to
+# the response being ready) and sends one row per minute, method, route and
+# status code (how many, total and slowest time, and a count per time bucket)
+# to POST /events/server-requests. Nothing about visitors is sent.
+#
+# The route is the framework's own pattern when the integration knows it
+# (Rails, Sinatra); otherwise numbers and IDs in the path become ":id", and
+# unmatched 404s are one route.
+
+module SiteGoose
+  class Performance
+    # Keep in step with the collector (expresscollect/Analytics/serverRequests.js).
+    BUCKET_EDGES_MS = [5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000].freeze
+    FLUSH_EVERY = 60
+    MAX_KEYS = 300
+    # Past MAX_KEYS new routes fold into "(other)"; past this nothing new is
+    # kept at all, whatever the requests carry.
+    HARD_MAX_KEYS = 400
+    # The method is the visitor's to choose: anything else is one "OTHER".
+    METHODS = %w[GET HEAD POST PUT PATCH DELETE OPTIONS JOB].freeze
+    MAX_ROWS = 500
+    NOT_FOUND = "/(not found)"
+    ID = /\A(\d+|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{16,})\z/i.freeze
+
+    # "/orders/123/items/9f8e..." -> "/orders/:id/items/:id"
+    def self.normalize_route(path)
+      path = SiteGoose.text(path).split(/[?#]/, 2)[0].to_s
+      path = "/" if path.empty?
+      parts = path.split("/", -1).map { |s| s.match?(ID) || (s.size >= 20 && s.match?(/\d/)) ? ":id" : s }
+      route = parts.join("/")
+      (route.start_with?("/") ? route : "/#{route}")[0, 200]
+    end
+
+    def self.method_of(method)
+      name = SiteGoose.text(method).upcase
+      return "GET" if name.empty?
+
+      METHODS.include?(name) ? name : "OTHER"
+    end
+
+    def self.bucket_of(ms)
+      BUCKET_EDGES_MS.index { |edge| ms <= edge } || BUCKET_EDGES_MS.size
+    end
+
+    attr_reader :rows, :server_name
+
+    # Options: server_name (default the host name).
+    def initialize(core, server_name: nil, **_)
+      @core = core
+      @server_name = SiteGoose.text(server_name || SiteGoose.hostname, 100)
+      @rows = {}
+      @lock = Mutex.new
+      @job = core.valid? ? Periodic.new(FLUSH_EVERY) { flush(everything: false); FLUSH_EVERY } : nil
+      Forking.register(self)
+    end
+
+    # In a forked worker the parent's rows are the parent's to send.
+    def after_fork_in_child
+      @lock = Mutex.new
+      @rows = {}
+    end
+
+    # One finished request. route: the framework's pattern, or "" to work it
+    # out from path. -> [method, route] as recorded (the database queries go
+    # under the same), or nil when Server speed is off.
+    def record(method, route, status, ms, path = "")
+      return unless @core.on?("performance")
+
+      status = status.to_i
+      route = SiteGoose.text(route)
+      route = status == 404 ? NOT_FOUND : Performance.normalize_route(path) if route.empty?
+      method = Performance.method_of(method)
+      minute = (Time.now.to_i / 60) * 60
+      ms = ms.to_f
+      @lock.synchronize do
+        key = [minute, method, route[0, 200], status]
+        key = [minute, method, "(other)", status] if !@rows.key?(key) && @rows.size >= MAX_KEYS
+        return key[1, 2] if !@rows.key?(key) && @rows.size >= HARD_MAX_KEYS
+
+        row = (@rows[key] ||= { count: 0, sum_ms: 0.0, max_ms: 0.0, buckets: Array.new(BUCKET_EDGES_MS.size + 1, 0) })
+        row[:count] += 1
+        row[:sum_ms] += ms
+        row[:max_ms] = ms if ms > row[:max_ms]
+        row[:buckets][Performance.bucket_of(ms)] += 1
+        key[1, 2]
+      end
+    end
+
+    def flush(everything: true)
+      current = (Time.now.to_i / 60) * 60
+      ready = @lock.synchronize do
+        taken = @rows.select { |k, _| everything || k[0] < current }
+        taken.each_key { |k| @rows.delete(k) }
+        taken
+      end
+      rows = ready.map do |k, v|
+        { "minute" => Time.at(k[0]).utc.iso8601(3), "method" => k[1], "route" => k[2], "status" => k[3],
+          "count" => v[:count], "sum_ms" => v[:sum_ms].round(1), "max_ms" => v[:max_ms].round(1), "buckets" => v[:buckets] }
+      end
+      rows.each_slice(MAX_ROWS) do |batch|
+        @core.request("performance", "/events/server-requests",
+                      body: { "server_name" => @server_name, "runtime" => RUNTIME, "rows" => batch })
+      end
+    end
+
+    def close
+      @job&.stop
+      Forking.unregister(self)
+      flush
+    end
+  end
+end