skadi 0.4.0.beta.1

data/lib/skadi/url.rb

@@ -0,0 +1,66 @@
+module Skadi
+  # Helper functions for generating and redacting URLs
+  module Url
+    # Formats the path for Skadi views. Note that the path here uses PATH_INFO, which does not include the query string or fragment.
+    # @param request [ActionDispatch::Request]
+    # @return [String]
+    def self.view_path_from_request(request)
+      path = +""
+
+      if Skadi.configuration.store_domain_in_views
+        path << request.host_with_port
+      end
+
+      # Normalise the path by removing any trailing slashes
+      path << ((request.path == "/" || request.path == "") ? "/" : request.path.chomp("/"))
+
+      path
+    end
+
+    # @param query_params [Hash, ActiveSupport::HashWithIndifferentAccess]
+    # @return [Hash]
+    def self.whitelist_query_params(query_params)
+      # Normalise the input to a Hash with symbolic keys
+      query_params = query_params.to_h.symbolize_keys
+
+      return query_params unless Skadi.configuration.use_query_param_whitelist
+
+      whitelist = Skadi.configuration.query_param_whitelist
+      return {} if whitelist.empty?
+
+      query_params.slice(*whitelist)
+    end
+
+    # Strips non-whitelisted query params and normalises URLs
+    # @param url [String]
+    # @return [String, nil]
+    def self.redact_and_normalise_url(url)
+      return nil unless url.is_a?(String) && url.present?
+
+      uri = URI.parse(url[0, Skadi.configuration.max_url_length])
+      return nil if uri.opaque
+
+      query_params = Rack::Utils.parse_nested_query(uri.query) if uri.query.present?
+      param_string = whitelist_query_params(query_params).to_query if query_params.present?
+
+      result = +""
+
+      # Only record interesting schemes, e.g. "android-app://"
+      result += "#{uri.scheme}://" if uri.scheme.present? && ![ "http", "https" ].include?(uri.scheme)
+
+      result += uri.host if uri.host.present?
+
+      # Only include port if it's non-standard
+      result << ":#{uri.port}" if uri.port != uri.default_port
+
+      # Normalise the trailing slash
+      result << ((uri.path == "" || uri.path == "/") ? "/" : uri.path.chomp("/")) unless uri.path.nil?
+
+      result << (param_string.present? ? "?#{param_string}" : "")
+
+      return result
+    rescue URI::InvalidURIError, Rack::QueryParser::ParameterTypeError, Rack::QueryParser::QueryLimitError
+      return nil
+    end
+  end
+end

data/lib/skadi/user_agent.rb

@@ -0,0 +1,458 @@
+module Skadi
+  # A minimal user agent parser, designed for speed rather than completeness, aiming to detect the most common browsers
+  # and operating systems.
+  #
+  # Why implement a custom user agent parsing script over an existing library?
+  #
+  # Firstly, to eliminate the transient dependencies consumers of this gem are exposed to.
+  #
+  # Secondly, existing libraries have relatively poor performance. Unlike other solutions, which use a large number of
+  # regular expressions to detect browsers, this parser tokenises the user agent and uses a lookup table to identify the
+  # majority of browsers with O(1) performance, using regular expressions as fallback. See the `user_agent_test.rb` file
+  # for performance statistics.
+  #
+  # Thirdly, the user agent parsing gems are years out of date. Modern browsers and modern bots are not properly
+  # detected, potentially skewing the analytics data collected.
+  class UserAgent
+    def initialize(user_agent)
+      @user_agent = (user_agent || "")[0, 2048]
+    end
+
+    def browser
+      parse_browser if @browser.nil?
+
+      @browser
+    end
+
+    def browser_version
+      parse_browser if @browser_version.nil?
+
+      @browser_version
+    end
+
+    def engine
+      parse_engine if @engine.nil?
+
+      @engine
+    end
+
+    def engine_version
+      parse_engine if @engine_version.nil?
+
+      @engine_version
+    end
+
+    def os
+      parse_os if @os.nil?
+
+      @os
+    end
+
+    def bot?
+      return @bot unless @bot.nil?
+
+      # If the user agent doesn't follow the normal browser patterns, we assume it's a bot
+      parse_browser if @browser.nil?
+      if @browser == "Unknown"
+        @bot = true
+
+        return true
+      end
+
+      @bot = detect_bot
+    end
+
+    def human? = !bot?
+
+    def to_h
+      {
+        browser: browser,
+        browser_version: browser_version,
+        engine: engine,
+        engine_version: engine_version,
+        os: os,
+        bot: bot?,
+      }
+    end
+
+    BROWSER_MATCHERS = [
+      {
+        regex: /Chrome\/(?<version>\d++).*WebView|; wv.*Chrome\/(?<version>\d++)/,
+        browser: "Chrome WebView",
+      },
+      {
+        regex: /Android.*Version\/(?<version>\d++)/,
+        browser: "Android Browser",
+        os: "Android",
+      },
+      {
+        regex: /Android.*Chrome\/(?<version>\d++)/,
+        browser: "Chrome for Android",
+        os: "Android",
+      },
+      {
+        regex: /(?:iOS|iPod|iPad|iPhone).+(?:CriOS|Chrome)\/(?<version>\d++)/,
+        browser: "Chrome for iOS",
+        os: "iOS",
+      },
+      {
+        regex: /(?:CriOS|Chrome)\/(?<version>\d++)/,
+        browser: "Chrome",
+      },
+      {
+        regex: /(?:iOS|iPod|iPad|iPhone).+Version\/(?<version>\d++)/,
+        browser: "Safari for iOS",
+        os: "iOS",
+      },
+      {
+        regex: /GSA\/(?<version>\d++)/,
+        browser: "GSA",
+      },
+      {
+        regex: /(?:iOS|iPod|iPad|iPhone).*Safari/,
+        browser: "Safari for iOS",
+        os: "iOS",
+      },
+      {
+        regex: /Version\/(?<version>\d++).*Safari/,
+        browser: "Safari",
+      },
+      {
+        regex: /Safari\//,
+        browser: "Safari",
+      },
+      {
+        regex: /WebKit\/(?<version>\d++)/,
+        browser: "WebKit",
+      },
+      {
+        regex: /Mozilla\/(?<version>\d++).*rv:(?<engine_version>\d++).*?Gecko\//,
+        browser: "Mozilla",
+        engine: "Gecko",
+      },
+    ]
+
+    BROWSER_TOKENS = {
+      "AlohaBrowser" => {
+        browser: "Aloha",
+      },
+      "Avast" => {
+        browser: "Avast Secure Browser",
+      },
+      "AVG" => {
+        browser: "AVG Secure Browser",
+      },
+      "baiduboxapp" => {
+        browser: "Baidu",
+      },
+      "BingSapphire" => {
+        browser: "Bing",
+      },
+      "Brave" => {},
+      "Chromium" => {
+        regex: /Chromium[\/ ](?<version>GOST|\d++)/,
+      },
+      "Ddg" => {
+        browser: "DuckDuckGo",
+      },
+      "DuckDuckGo" => {},
+      "Ecosia" => {
+        regex: /Ecosia ios@(?<version>\d++)/,
+        os: "iOS",
+      },
+      "Edg" => [
+        {
+          regex: /(?:iOS|iPod|iPad|iPhone).*Edg\/(?<version>\d++)/,
+          browser: "Edge for iOS",
+          os: "iOS",
+        },
+        {
+          regex: /Android.*Edg\/(?<version>\d++)/,
+          browser: "Edge for Android",
+          os: "Android",
+        },
+        {
+          regex: /Edg\/(?<version>\d++)/,
+          browser: "Edge",
+        },
+      ],
+      "EdgA" => {
+        browser: "Edge for Android",
+        os: "Android",
+      },
+      "Edge" => {},
+      "EdgiOS" => {
+        browser: "Edge for iOS",
+        os: "iOS",
+      },
+      "Electron" => {},
+      "FBAV" => {
+        browser: "Facebook",
+      },
+      "Firefox" => [
+        {
+          regex: /(?<browser>PaleMoon|Waterfox)\/(?<version>\d++)/,
+        },
+        {
+          regex: /(?:iOS|iPod|iPad|iPhone).*Firefox\/(?<version>\d++)/,
+          browser: "Firefox for iOS",
+          os: "iOS",
+        },
+        {
+          regex: /Android.*Firefox\/(?<version>\d++)/,
+          browser: "Firefox for Android",
+          os: "Android",
+        },
+        {
+          regex: /Firefox\/(?<version>\d++)/,
+          browser: "Firefox",
+        },
+      ],
+      "FxiOS" => {
+        browser: "Firefox for iOS",
+      },
+      "HeadlessChrome" => {
+        browser: "Chrome Headless",
+      },
+      "HeyTapBrowser" => {
+        browser: "HeyTap",
+      },
+      "HuaweiBrowser" => {
+        browser: "Huawei Browser",
+      },
+      "Instagram" => {
+        regex: /Instagram[\/ ](?<version>\d++)/,
+      },
+      "Iron" => {
+        regex: /Chrome\/(?<version>\d++).*?Iron/,
+      },
+      "KAKAOTALK" => {
+        regex: /KAKAOTALK[\/ ](?<version>\d++)/,
+      },
+      "Konqueror" => {},
+      "Line" => {},
+      "LinkedInApp" => {
+        regex: /\[LinkedInApp\]\/(?<version>\d++)/,
+        browser: "LinkedIn",
+      },
+      "Maxthon" => {},
+      "MicroMessenger" => {
+        browser: "WeChat",
+      },
+      "MiuiBrowser" => {
+        browser: "MIUI Browser",
+      },
+      "MQQBrowser" => {},
+      "MSIE" => {
+        regex: /MSIE (?<version>\d++)(?>.*(?<engine>Trident)\/(?<engine_version>\d++))?|(?<engine>Trident)\/(?<engine_version>\d++).*rv[: ](?<version>\d++)/,
+        browser: "IE",
+        os: "Windows",
+      },
+      "musical" => {
+        regex: /musical_ly_(?<version>\d++)/,
+        browser: "TikTok",
+      },
+      "Norton" => {
+        browser: "Norton Private Browser",
+      },
+      "OculusBrowser" => {
+        browser: "Oculus Browser",
+      },
+      "Opera" => [
+        {
+          regex: /Opera Mini[\/ ](?<version>\d++)/,
+          browser: "Opera Mini",
+        },
+        {
+          regex: /Opera(?>.*Version)?[\/ ](?<version>\d++)/,
+          browser: "Opera",
+        },
+      ],
+      "OPR" => {
+        browser: "Opera",
+      },
+      "OPT" => {
+        browser: "Opera Touch",
+      },
+      "OPX" => {
+        browser: "Opera GX",
+      },
+      "PaleMoon" => {},
+      "QQBrowser" => {},
+      "QuarkPC" => {
+        browser: "Quark",
+      },
+      "SamsungBrowser" => {
+        browser: "Samsung Internet",
+      },
+      "SeaMonkey" => {},
+      "Silk" => {},
+      "Snapchat" => {},
+      "TikTokLIVEStudio" => {},
+      "Trident" => {
+        regex: /MSIE (?<version>\d++).*Trident\/(?<engine_version>\d++)|Trident\/(?<engine_version>\d++).*rv[: ](?<version>\d++)/,
+        browser: "IE",
+        engine: "Trident",
+        os: "Windows",
+      },
+      "Twitter" => {
+        regex: /Twitter for iPhone\/(?<version>\d++)/,
+      },
+      "UCBrowser" => {},
+      "VivoBrowser" => {
+        browser: "Vivo Browser",
+      },
+      "Whale" => {},
+      "YaBrowser" => {
+        browser: "Yandex",
+      },
+      "YaSearchBrowser" => {
+        browser: "Yandex",
+      },
+    }
+
+    # Normalise the BROWSER_TOKENS hash with defaults, and ensure they are all arrays
+    BROWSER_TOKENS.each_pair do |key, options|
+      next unless options.is_a?(Hash)
+
+      options[:regex] ||= /#{key}\/(?<version>\d++)/
+      options[:browser] ||= key
+
+      # Normalise into an array so we don't have to check the type during runtime
+      BROWSER_TOKENS[key] = [ options ]
+    end
+
+    private def parse_browser
+      # First we check if any of the UA tokens exist as keys to our browser token list (fast!)
+      user_agent_tokens.each do |token|
+        next unless BROWSER_TOKENS.key?(token)
+
+        # We then use the matcher to extract the version from the full user agent
+        BROWSER_TOKENS[token].each do |matcher|
+          return if run_matcher(matcher) # rubocop:disable Lint/NonLocalExitFromIterator
+        end
+      end
+
+      # Then, if we don't get a match, we run the full list of fallback matchers against the user agent (slow!)
+      BROWSER_MATCHERS.each do |matcher|
+        return if run_matcher(matcher) # rubocop:disable Lint/NonLocalExitFromIterator
+      end
+
+      # Finally, falling back to unknown values for the browser variables
+      @browser = "Unknown"
+      @browser_version = "Unknown"
+    end
+
+    # Runs a matcher hash against the user agent, srtting relevant instance variables. Returns true if a match was found.
+    private def run_matcher(matcher)
+      match = matcher[:regex].match(@user_agent)
+
+      if match
+        named_captures = match.named_captures
+
+        @browser = named_captures["browser"] || matcher[:browser] || "Unknown"
+        @browser_version = named_captures["version"] || matcher[:browser_version] || "Unknown"
+        @engine = named_captures["engine"] || matcher[:engine]
+        @engine_version = named_captures["engine_version"]
+        @os = named_captures["os"] || matcher[:os]
+
+        return true
+      end
+
+      false
+    end
+
+    ENGINE_MATCHERS = [
+      {
+        regex: /AppleWebKit\/537\.36.*Edge\/(?<version>1[2-8])\./,
+        engine: "EdgeHTML",
+      },
+      {
+        regex: /AppleWebKit\/537\.36.*Chrome\/(?<version>\d++)/,
+        engine: "Blink",
+      },
+      {
+        regex: /(?<engine>WebKit|Presto|Trident|Goanna)\/(?<version>\d++)/,
+      },
+      {
+        regex: /rv:(?<version>\d++).*?Gecko\/\d++/,
+        engine: "Gecko",
+      },
+    ]
+
+    private def parse_engine
+      # Since there are only 4 engine matchers, there is little performance to be gained by using the token approach to parsing
+      ENGINE_MATCHERS.each do |matcher|
+        match = matcher[:regex].match(@user_agent)
+
+        if match
+          named_captures = match.named_captures
+
+          @engine = named_captures["engine"] || matcher[:engine] || "Unknown"
+          @engine_version = named_captures["version"] || "Unknown"
+
+          return # rubocop:disable Lint/NonLocalExitFromIterator
+        end
+      end
+
+      @engine = "Unknown"
+      @engine_version = "Unknown"
+    end
+
+    OS_TOKENS = {
+      "CFNetwork" => "iOS",
+      "CrOS" => "Chrome OS",
+      "Fedora" => "Fedora",
+      "Gentoo" => "Gentoo",
+      "HarmonyOS" => "HarmonyOS",
+      "iPad" => "iOS",
+      "iPhone" => "iOS",
+      "iPod" => "iOS",
+      "Mac" => "macOS",
+      "Ubuntu" => "Ubuntu",
+      "Windows" => "Windows",
+    }
+
+    private def parse_os
+      linux_fallback = false
+      android_fallback = false
+      user_agent_tokens.each do |token|
+        if OS_TOKENS.key?(token)
+          @os = OS_TOKENS[token]
+
+          return # rubocop:disable Lint/NonLocalExitFromIterator
+        end
+
+        android_fallback ||= token == "Android"
+        linux_fallback ||= token == "Linux"
+      end
+
+      # HarmonyOS UAs can contain Android, and Android UAs can contain "Linux" so we need to do these in a specific order
+      @os ||= "Android" if android_fallback
+      @os ||= "Linux" if linux_fallback
+      @os ||= "Unknown"
+    end
+
+    BOT_GLOBAL_MATCHERS = %w[bot crawl scan spider]
+
+    BOT_WORD_SET = Set.new(%w[AGENT Agent AppInsights ArchiveBox Archiver Archiving BingPreview BrandVerity Butterfly Charlotte Checkly Claude CloudFlare Cloudflare Code Collapsify CookieHubVerify Criticalcss Daily DareBoost DatadogSynthetics Datanyze Devin Dlc FeedBurner Feeder Feedly FlipboardProxy Fluid Foregenix GTmetrix GeedoProductSearch GeedoShopProductFinder Google GoogleAgent GoogleImageProxy GotSiteMonitor Hardenize HeadlessChrome Hotjar Inspector Lighthouse LinkTiger Mail Manus MarketGoo MarketingMiner MetaIAB Miniature MonitoRSS Monitor Netcraft NewRelicSynthetics NewsBlur NewsNow Newsify Nitro OpenGraph Optimizer PTST PWABuilderHttpAgent Perplexity PingdomTMS Playwright Preview PrintFriendly Puppeteer Readable RevvimGort Rigor SQWatcher Scope3 SecurityHeaders Selenium SeoSiteCheckup Silktide Sindup Siteimprove Specificfeeds Sucuri TestLocally ThousandEyes Trae YLT ZoteroTranslationServer adbeat agent archiver archiving brandverity butterfly claude cloudflare code contentkingapp deadlinkchecker devin europarchive feeder feedly google img2dataset infegy mail mailservertest2023 marketingminer mirrorweb monitor nbertaupete95 netcraft newsai newsblur newsify opencode opengraph oupwis perplexity preview retrevo scope3 scraping seositecheckup sitebulb slider splash sqwatcher turingos ubermetrics uptimedoctor watchTowr webresearch websitepulse woorankreview xmco])
+
+    BOT_FALLBACK_MATCHERS = [ "AP3A.240617.008" ]
+
+    private def detect_bot
+      return true if user_agent_tokens.any? { |it| BOT_WORD_SET.include? it }
+
+      user_agent_downcase = @user_agent.downcase
+      return true if BOT_GLOBAL_MATCHERS.any? { |it| user_agent_downcase.include? it }
+
+      return true if BOT_FALLBACK_MATCHERS.any? { |it| @user_agent.include? it }
+
+      false
+    end
+
+    # Splits the string into alphanumeric sequences of length 3 or more
+    private def user_agent_tokens
+      @user_agent_tokens ||= @user_agent.tr("^a-zA-Z0-9", " ").split.keep_if { |it| it.length >= 3 }
+    end
+  end
+end

data/lib/skadi/version.rb

@@ -0,0 +1,3 @@
+module Skadi
+  VERSION = "0.4.0.beta.1"
+end

data/lib/skadi.rb

@@ -0,0 +1,26 @@
+require "active_support"
+require "active_support/concern"
+require "active_support/core_ext"
+
+require "action_dispatch/http/request"
+
+require_relative "skadi/anonymity_set"
+require_relative "skadi/analytics"
+require_relative "skadi/configuration"
+require_relative "skadi/controller_delegate"
+require_relative "skadi/cookie_manager"
+require_relative "skadi/engine"
+require_relative "skadi/url"
+require_relative "skadi/user_agent"
+require_relative "skadi/version"
+
+module Skadi
+  # @return [Skadi::Configuration] The Skadi configuration
+  mattr_accessor :configuration, default: Configuration.new
+
+  # @yieldparam config [Skadi::Configuration]
+  # @return [void]
+  def self.configure
+    yield(configuration)
+  end
+end