sitegoose 0.3.3

data/lib/sitegoose/posture.rb

@@ -0,0 +1,272 @@
+# frozen_string_literal: true
+
+# How the server is set up (protocol in sdk/protocols/posture.md): the
+# settings that leave the app open (Rails', when it's a Rails app), whether it
+# runs as root, code anyone on the machine can write, and files in the folders
+# it serves to the web that should never be public. Sent to POST
+# /events/posture a few minutes after start and then once a day (the
+# collector's `every`), so Security -> Security check can say what to change.
+#
+# It only reads: nothing here changes a setting, a permission or a file, and
+# no file's contents are sent -- paths, sizes, times and short values only.
+# The walk is bounded and runs in a background thread, never in a request;
+# with no folder served to the web there is nothing to walk and only the
+# settings are reported.
+
+module SiteGoose
+  class Posture
+    FIRST = 180
+    RETRY_EVERY = 60 * 60
+    EVERY = 24 * 60 * 60
+    MAX_CHECKS = 200
+    MAX_EXPOSURE = 50
+    MAX_ENTRIES = 20_000
+    MAX_DEPTH = 6
+    MAX_PERM_CHECKS = 3000
+    SKIP_DIRS = %w[.git .svn .hg node_modules vendor cache .cache .history __pycache__].freeze
+    VCS_DIRS = %w[.git .svn .hg .bzr].freeze
+    CODE_FILE = /\.(py|pyw|js|cjs|mjs|php[0-9]?|phtml|rb|erb|pl|sh)\z/i.freeze
+    # Files that shouldn't be downloadable, by name; first match wins.
+    EXPOSURE = [
+      ["database_dump", /\.sql(\.(gz|bz2|xz|zip))?\z/i],
+      ["private_key", /\A(id_rsa|id_dsa|id_ecdsa|id_ed25519)\z|\.(pem|key|p12|pfx|jks)\z/i],
+      ["env", /\A\.env(\z|\.)|\Aenv\.(ini|txt)\z/i],
+      ["archive", /\.(zip|tar|tgz|tar\.gz|tar\.bz2|rar|7z)\z/i],
+      ["backup", /\.(bak|old|orig|save|swp|copy|backup)\z|~\z|\.(py|rb|yml|json|conf|config|ini|ya?ml)\.(bak|old|orig|save|txt)\z/i],
+      ["info", /\A(phpinfo|info|adminer)\.php\z/i],
+      ["log", /\A(debug|error|access)[-_.]?log\z|\.log\z/i]
+    ].freeze
+    # What a secret looks like when nobody set one.
+    INSECURE_KEYS = %w[changeme secret your-secret-key put-your-secret-key-here].freeze
+
+    def self.kind_of(name)
+      EXPOSURE.each { |kind, pattern| return kind if pattern.match?(name) }
+      nil
+    end
+
+    # A folder inside another folder on the list is already covered: without
+    # this, every file under it would be reported twice.
+    def self.without_nested(roots)
+      unique = {}
+      roots.each { |root| unique[root["dir"]] ||= root }
+      unique.values.reject do |root|
+        unique.keys.any? { |other| other != root["dir"] && "#{root['dir']}/".start_with?("#{other}/") }
+      end
+    end
+
+    # [{"dir", "prefix"}]: prefix is where the web serves the folder, so the
+    # path we report is the one the dashboard can ask the site for. Not told:
+    # Rails' public/ (served at /), when it's a Rails app and it's there.
+    def self.roots_of(given)
+      roots = []
+      Array(given).each do |entry|
+        folder = entry.is_a?(Hash) ? (entry[:dir] || entry["dir"]) : entry
+        next if folder.nil? || folder.to_s.empty?
+
+        prefix = entry.is_a?(Hash) ? (entry[:prefix] || entry["prefix"]).to_s.gsub(%r{\A/+|/+\z}, "") : ""
+        roots << { "dir" => File.expand_path(folder.to_s), "prefix" => prefix }
+      end
+      return without_nested(roots) unless roots.empty?
+
+      folder = rails_public
+      roots << { "dir" => File.expand_path(folder), "prefix" => "" } if folder && File.directory?(folder)
+      without_nested(roots)
+    end
+
+    def self.rails_app
+      return nil unless defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
+
+      ::Rails.application
+    rescue StandardError
+      nil
+    end
+
+    def self.rails_public
+      return nil unless rails_app
+
+      ::Rails.public_path.to_s
+    rescue StandardError
+      nil
+    end
+
+    def self.check(checks, id, failed, value = nil)
+      row = { "id" => id, "state" => failed.nil? ? "unknown" : (failed ? "fail" : "pass") }
+      row["value"] = value.to_s[0, 200] if value && !value.to_s.empty?
+      checks << row
+    end
+
+    # The Rails settings that decide how much a hole in the site is worth.
+    def self.rails_checks
+      app = rails_app
+      return [] unless app
+
+      config = app.config
+      checks = []
+      env = ::Rails.env.to_s
+      check(checks, "app.rails_env", env != "production", env)
+      # Full error pages, with code and backtraces, for every visitor.
+      check(checks, "app.rails_debug", config.consider_all_requests_local ? true : false)
+      key = begin
+        app.secret_key_base.to_s
+      rescue StandardError
+        nil
+      end
+      check(checks, "app.secret_key", key.nil? ? nil : (key.size < 32 || INSECURE_KEYS.any? { |mark| key.downcase.include?(mark) }))
+      hosts = config.respond_to?(:hosts) ? Array(config.hosts) : nil
+      unless hosts.nil?
+        # An empty list switches host checking off: any Host header is served.
+        shown = hosts.first(10).map { |h| h.respond_to?(:source) ? h.source : h.to_s }
+        check(checks, "app.allowed_hosts", hosts.empty? || hosts.any? { |h| h.to_s == "*" }, shown.join(", "))
+      end
+      session = config.respond_to?(:session_options) ? (config.session_options || {}) : {}
+      check(checks, "app.session_cookie_httponly", session[:httponly] == false)
+      # Cookie flags only matter once the site is on HTTPS, which these say it is.
+      force_ssl = config.respond_to?(:force_ssl) && config.force_ssl ? true : false
+      if force_ssl || session[:secure]
+        check(checks, "app.session_cookie_secure", !(force_ssl || session[:secure]))
+        check(checks, "app.https_redirect", !force_ssl)
+      end
+      checks
+    rescue StandardError
+      checks || []
+    end
+
+    def self.app_checks
+      checks = rails_checks
+      # Running the web app as root turns any hole in it into the whole machine.
+      check(checks, "server.runs_as_root", Process.uid.zero?) if Process.respond_to?(:uid)
+      checks
+    end
+
+    # One bounded walk per root: [exposure, writable or nil, examples, truncated].
+    def self.walk(roots)
+      exposure = []
+      examples = []
+      writable = checked = entries = 0
+      roots.each do |root|
+        queue = [[root["dir"], 0]]
+        until queue.empty?
+          folder, depth = queue.shift
+          names = begin
+            Dir.children(folder)
+          rescue SystemCallError
+            next
+          end
+          names.each do |name|
+            entries += 1
+            return [exposure, checked.positive? ? writable : nil, examples, true] if entries > MAX_ENTRIES
+
+            path = File.join(folder, name)
+            info = begin
+              File.lstat(path)
+            rescue SystemCallError
+              next
+            end
+            next if info.symlink?
+
+            relative = [root["prefix"], path.delete_prefix("#{root['dir']}/")].reject(&:empty?).join("/")
+            if info.directory?
+              if VCS_DIRS.include?(name)
+                exposure << { "path" => "#{relative}/config", "kind" => "vcs" } if exposure.size < MAX_EXPOSURE
+                next
+              end
+              queue << [path, depth + 1] if depth + 1 <= MAX_DEPTH && !SKIP_DIRS.include?(name)
+              next
+            end
+            next unless info.file?
+
+            kind = kind_of(name)
+            if kind && exposure.size < MAX_EXPOSURE
+              exposure << { "path" => relative, "kind" => kind, "size" => info.size, "mtime" => info.mtime.utc.iso8601(3) }
+            end
+            next unless checked < MAX_PERM_CHECKS && CODE_FILE.match?(name)
+
+            checked += 1
+            if info.world_writable?
+              writable += 1
+              examples << relative if examples.size < 3
+            end
+          end
+        end
+      end
+      [exposure, checked.positive? ? writable : nil, examples, false]
+    end
+
+    # Options: roots (the folders the web serves, as "public" or
+    # {dir: ..., prefix: "assets"}; default: Rails' public/), host (the
+    # server's name), checks (a callable returning more checks of your own).
+    def initialize(core, roots: nil, host: nil, checks: nil, **_)
+      @core = core
+      @given = roots
+      @roots = nil
+      @extra = checks
+      @host = SiteGoose.text(host.nil? ? SiteGoose.hostname : host).gsub(/[^A-Za-z0-9._-]/, "-")[0, 100]
+      @host = "server" if @host.empty?
+      @job = core.valid? ? Periodic.new(FIRST) { send_report } : nil
+    end
+
+    # Worked out on first use, once the app (Rails above all) has booted.
+    def roots
+      @roots ||= Posture.roots_of(@given)
+    end
+
+    # What would be reported now (also used by the job and by tests).
+    def report
+      exposure, writable, examples, truncated = Posture.walk(roots)
+      checks = Posture.app_checks
+      unless writable.nil?
+        row = { "id" => "server.code_writable", "state" => writable.positive? ? "fail" : "pass" }
+        row["value"] = "#{writable} files, e.g. #{examples.join(', ')}" if writable.positive?
+        checks << row
+      end
+      begin
+        extra = @extra.respond_to?(:call) ? @extra.call : []
+        Array(extra).each do |c|
+          next unless c.is_a?(Hash)
+
+          c = c.to_h { |k, v| [k.to_s, v] }
+          next unless c["id"] && %w[pass fail unknown].include?(c["state"].to_s)
+
+          row = { "id" => c["id"].to_s.downcase[0, 60], "state" => c["state"].to_s }
+          row["value"] = c["value"].to_s[0, 200] if c["value"]
+          checks << row
+        end
+      rescue StandardError
+        nil
+      end
+      body = {
+        "site_id" => @core.site_id,
+        "host" => @host,
+        "at" => SiteGoose.iso_now,
+        "checks" => checks.first(MAX_CHECKS),
+        "facts" => { "agent" => "ruby/#{RUBY_VERSION}" }
+      }
+      unless roots.empty?
+        # Left out when there's no folder to walk: the dashboard then says
+        # this wasn't checked, instead of taking an empty walk for a clean one.
+        body["exposure"] = exposure.first(MAX_EXPOSURE)
+        body["facts"]["webroot"] = roots[0]["dir"][0, 100]
+      end
+      body["truncated"] = true if truncated
+      body
+    end
+
+    def send_report
+      # The set-up checks ride the software inventory switch.
+      return RETRY_EVERY unless @core.on?("inventory")
+
+      res = @core.request("posture", "/events/posture", auth: :bearer, body: report)
+      return RETRY_EVERY if res.nil? || !res.ok?
+
+      answer = res.json
+      every = answer.is_a?(Hash) && answer["every"].is_a?(Integer) && answer["every"].positive? ? answer["every"] : EVERY
+      [RETRY_EVERY, every].max
+    end
+
+    def flush; end
+
+    def close
+      @job&.stop
+    end
+  end
+end

data/lib/sitegoose/proxy.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+# First-party proxy: serving the tracker from the site's own domain, so ad
+# blockers don't hide visitors.
+#
+# Blockers stop analytics by recognising the address a script comes from and
+# what it is called ("track.js", "collect"). This app is already a server on
+# the site's domain, so the package serves both itself:
+#
+#     <prefix>/js/<rest>   ->  <endpoint>/<rest>                  (tracker files)
+#     <prefix>/api/<rest>  ->  <endpoint>/collectanalytics/<rest> (the collector)
+#
+# The same two rules as the hand-written Nginx/Cloudflare ones, so one install
+# check covers both. The prefix comes from the site secret: not guessable from
+# the page, and the same every time, so a browser's cache and a CDN in front
+# stay valid. It answers before every other module: a visitor's beacon is our
+# own traffic, not a page of the site, so it isn't counted, timed or rate-limited.
+
+require "digest"
+require "net/http"
+require "uri"
+
+module SiteGoose
+  class Proxy
+    # The tracker files, by name: anything else is refused rather than forwarded.
+    FILES = %w[track.js s.js snapshot.js replay.js survey.js banner.js leadform.js notice.js editor.js picker.js
+               feedback.js shield.js widget.js share.js].freeze
+    # A pinned version never changes; /v<major>/ follows new releases, so it is
+    # held for an hour, the same as the collector serves it with.
+    TTL_PINNED_S = 7 * 24 * 60 * 60
+    TTL_MAJOR_S = 60 * 60
+    MAX_BODY = 6 * 1024 * 1024
+    # The collector reads these; everything else, cookies above all, stays here.
+    PASS_HEADERS = %w[content-type user-agent accept-language referer origin sec-fetch-site sec-ch-ua
+                      sec-ch-ua-mobile sec-ch-ua-platform].freeze
+    SAFE_PATH = %r{\A[A-Za-z0-9/_.-]{1,120}\z}.freeze
+    FILE_PATH = %r{\Av(\d+(?:\.\d+\.\d+)?)/([a-z]+\.js)\z}.freeze
+    # These run on the app's own request threads, so they are short, and a
+    # collector that stops answering is left alone for a while rather than
+    # costing every beacon a timeout: a black-holed collector must not use
+    # up Puma's threads.
+    CONNECT_TIMEOUT = 1
+    READ_TIMEOUT = 2
+    BREAKER_FAILURES = 3
+    BREAKER_OPEN_S = 30
+    # A tracker file that couldn't be fetched isn't asked for again this soon.
+    FILE_RETRY_S = 30
+
+    def self.not_found
+      [404, { "content-type" => "text/plain; charset=utf-8" }, "Not found"]
+    end
+
+    def self.prefix_for(secret)
+      Digest::SHA256.hexdigest("proxy|#{secret}")[0, 8]
+    end
+
+    # Sent as X-Proxy-Key, so the collector believes the visitor addresses
+    # this proxy passes on: anyone can write X-Forwarded-For. Worked out from
+    # the secret as the collector does (expresscollect/utils/proxyKey.js).
+    def self.key_for(secret)
+      Digest::SHA256.hexdigest("proxykey|#{secret}")[0, 32]
+    end
+
+    attr_reader :prefix, :key
+
+    # prefix: the path it answers on (default: from the secret).
+    def initialize(core, prefix: nil, enabled: true, **_)
+      @core = core
+      @enabled = enabled
+      @prefix = (prefix || Proxy.prefix_for(core.secret)).to_s.gsub(/[^A-Za-z0-9_-]/, "")
+      @key = core.secret.empty? ? "" : Proxy.key_for(core.secret)
+      # One copy of each tracker file, so a repeat visitor costs no upstream
+      # request. A handful of small files: it never grows.
+      @files = {}
+      @failed = {}
+      @lock = Mutex.new
+      @failures = 0
+      @open_until = 0.0
+    end
+
+    # True while the collector is being left alone (after BREAKER_FAILURES
+    # failures in a row). When the pause ends one request tries again; the
+    # others keep the short answer until it has.
+    def resting?
+      now = SiteGoose.clock
+      @lock.synchronize do
+        next false if @failures < BREAKER_FAILURES
+        next true if now < @open_until
+
+        @open_until = now + READ_TIMEOUT + CONNECT_TIMEOUT
+        false
+      end
+    end
+
+    def worked
+      @lock.synchronize { @failures = 0 }
+    end
+
+    def failed
+      @lock.synchronize do
+        @failures += 1
+        @open_until = SiteGoose.clock + BREAKER_OPEN_S if @failures >= BREAKER_FAILURES
+      end
+    end
+
+    def on?
+      @enabled && @core.valid? && !@prefix.empty? && @core.on?("proxy")
+    end
+
+    # The tracker's address on this site, for the page's script tag.
+    def script_url(version = "1")
+      "/#{@prefix}/js/v#{version}/s.js"
+    end
+
+    # Where visits are sent, for the tag's data-collect.
+    def collect_url
+      "/#{@prefix}/api/e"
+    end
+
+    # [kind ("js" | "api" | "bad"), the rest] when the address is ours, else nil.
+    def match(path)
+      return nil unless on?
+
+      head = "/#{@prefix}/"
+      path = path.to_s
+      return nil unless path.start_with?(head)
+
+      rest = path[head.size..]
+      %w[js api].each do |kind|
+        next unless rest.start_with?("#{kind}/")
+
+        tail = rest[(kind.size + 1)..]
+        return [kind, tail] if !tail.empty? && !tail.include?("..") && tail.match?(SAFE_PATH)
+
+        break
+      end
+      ["bad", ""]
+    end
+
+    # [status, headers, body]. Blocking (up to the timeout).
+    def handle(kind, rest, method: "GET", query: "", headers: {}, body: nil, client_ip: "")
+      return file(rest) if kind == "js"
+      return Proxy.not_found unless kind == "api"
+
+      collector(rest, method, query, headers || {}, body, client_ip)
+    end
+
+    def fetch(url, method: "GET", headers: {}, body: nil)
+      uri = URI.parse(url)
+      klass = method == "POST" ? Net::HTTP::Post : Net::HTTP::Get
+      req = klass.new(uri.request_uri, headers)
+      req.body = body if body && method == "POST"
+      Forking.sending do
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = CONNECT_TIMEOUT
+        http.read_timeout = READ_TIMEOUT
+        http.write_timeout = READ_TIMEOUT if http.respond_to?(:write_timeout=)
+        http.start { |h| h.request(req) }
+      end
+    end
+
+    # Any version number fits the file pattern, so the list is kept short.
+    def remember_failure(rest)
+      @lock.synchronize do
+        @failed.delete(rest)
+        @failed.shift while @failed.size >= 200
+        @failed[rest] = SiteGoose.clock
+      end
+    end
+
+    def file(rest)
+      found = FILE_PATH.match(rest)
+      return Proxy.not_found if found.nil? || !FILES.include?(found[2])
+
+      pinned = found[1].include?(".")
+      held = @lock.synchronize { @files[rest] }
+      body = nil
+      body = held[0] if held && held[1] > Time.now.to_f - (pinned ? TTL_PINNED_S : TTL_MAJOR_S)
+      recently_failed = @lock.synchronize { (at = @failed[rest]) && at > SiteGoose.clock - FILE_RETRY_S }
+      if body.nil? && !recently_failed && !resting?
+        begin
+          answer = fetch("#{@core.endpoint}/#{rest}")
+          fetched = answer.body.to_s
+          answer.code.to_i >= 500 ? failed : worked
+          if answer.code.to_i == 200 && fetched.bytesize > 500 && fetched.bytesize <= MAX_BODY
+            body = fetched
+            @lock.synchronize do
+              @files[rest] = [body, Time.now.to_f]
+              @failed.delete(rest)
+            end
+          else
+            remember_failure(rest)
+          end
+        rescue StandardError, Timeout::Error => e
+          failed
+          remember_failure(rest)
+          @core.log("proxy", "could not fetch #{rest}: #{e.message}")
+        end
+      end
+      # Upstream is unreachable: an old copy beats no tracker at all.
+      body = held[0] if body.nil? && held
+      return [502, {}, ""] if body.nil?
+
+      [200, {
+        "content-type" => "application/javascript; charset=utf-8",
+        "cache-control" => pinned ? "public, max-age=31536000, immutable" : "public, max-age=3600",
+        "x-content-type-options" => "nosniff"
+      }, body]
+    end
+
+    def collector(rest, method, query, headers, body, client_ip)
+      method = (method.to_s.empty? ? "GET" : method.to_s).upcase
+      if method == "OPTIONS"
+        # Same origin as the page, so a preflight shouldn't happen; answer it
+        # cheaply rather than passing it on.
+        return [204, { "access-control-allow-methods" => "GET, POST, OPTIONS", "access-control-allow-headers" => "Content-Type",
+                       "access-control-max-age" => "86400" }, ""]
+      end
+      return Proxy.not_found unless %w[GET POST].include?(method)
+      return [413, {}, ""] if body && body.bytesize > MAX_BODY
+
+      # The visitor's address: the collector needs it for the country, the
+      # daily visitor ID and the per-visitor rate limit.
+      out = { "X-Forwarded-For" => client_ip.to_s }
+      out["X-Proxy-Key"] = @key unless @key.empty?
+      PASS_HEADERS.each do |name|
+        value = headers[name]
+        next if value.nil? || value.to_s.empty?
+
+        out[name.split("-").map(&:capitalize).join("-")] = value.to_s.delete("\r\n")
+      end
+      url = "#{@core.endpoint}/collectanalytics/#{rest}#{query.to_s.empty? ? '' : "?#{query}"}"
+      # Left alone for now: answered straight away; the tracker treats a
+      # failed beacon as lost and carries on.
+      return [502, { "retry-after" => BREAKER_OPEN_S.to_s }, ""] if resting?
+
+      begin
+        answer = fetch(url, method: method, headers: out, body: method == "POST" ? body.to_s : nil)
+        answer.code.to_i >= 500 ? failed : worked
+        passed = {}
+        %w[content-type cache-control retry-after].each do |name|
+          value = answer[name]
+          passed[name] = value if value && !value.empty?
+        end
+        text = answer.body.to_s
+        [answer.code.to_i, passed, text.bytesize > MAX_BODY ? text.byteslice(0, MAX_BODY) : text]
+      rescue StandardError, Timeout::Error => e
+        # The tracker already treats a failed beacon as lost and carries on.
+        failed
+        @core.log("proxy", "could not reach the collector: #{e.message}")
+        [502, {}, ""]
+      end
+    end
+
+    def flush; end
+
+    def close; end
+  end
+end