aikido-zen 0.1.1-x86_64-darwin → 1.0.0.pre.beta.1-x86_64-darwin

data/lib/aikido/zen/detached_agent/agent.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "drb/drb"
+require "drb/unix"
+require_relative "front_object"
+require_relative "../background_worker"
+
+module Aikido::Zen::DetachedAgent
+  # Agent that runs in forked processes. It communicates with the parent process to dRB
+  # calls. It's in charge of schedule and send heartbeats to the *parent process*, to be
+  # later pushed.
+  #
+  # heartbeat & polling interval are configured to 10s , because they are connecting with
+  # parent process. We want to have the freshest data.
+  #
+  # It's possible to use `extend Forwardable` here for one-line forward calls to the
+  # @detached_agent_front object. Unfortunately, the methods to be called are
+  # created at runtime by `DRbObject`, which leads to an ugly warning about
+  # private methods after the delegator is bound.
+  class Agent
+    attr_reader :worker
+
+    def initialize(
+      heartbeat_interval: 10,
+      polling_interval: 10,
+      config: Aikido::Zen.config,
+      collector: Aikido::Zen.collector,
+      worker: Aikido::Zen::Worker.new(config: config)
+    )
+      @config = config
+      @heartbeat_interval = heartbeat_interval
+      @polling_interval = polling_interval
+      @worker = worker
+      @collector = collector
+      @detached_agent_front = DRbObject.new_with_uri(config.detached_agent_socket_path)
+      @has_forked = false
+      schedule_tasks
+    end
+
+    def send_heartbeat(at: Time.now.utc)
+      return unless @collector.stats.any?
+
+      heartbeat = @collector.flush(at: at)
+      @detached_agent_front.send_heartbeat_to_parent_process(heartbeat.as_json)
+    end
+
+    private def schedule_tasks
+      # For heartbeats is correct to send them from parent or child process. Otherwise, we'll lose
+      # stats made by the parent process.
+      @worker.every(@heartbeat_interval, run_now: false) { send_heartbeat }
+
+      # Runtime_settings fetch must happens only in the child processes, otherwise, due to
+      # we are updating the global runtime_settings, we could have an infinite recursion.
+      if @has_forked
+        @worker.every(@polling_interval) do
+          Aikido::Zen.runtime_settings = @detached_agent_front.updated_settings
+          @config.logger.debug "Updated runtime settings after polling from child process #{Process.pid}"
+        end
+      end
+    end
+
+    def calculate_rate_limits(request)
+      @detached_agent_front.calculate_rate_limits(request.route, request.ip, request.actor.to_json)
+    end
+
+    # Every time a fork occurs (a new child process is created), we need to start
+    # a DRb service in a background thread within the child process. This service
+    # will manage the connection and handle resource cleanup.
+    def handle_fork
+      @has_forked = true
+      DRb.start_service
+      # we need to ensure that there are not more jobs in the queue, but
+      # we reuse the same object
+      @worker.restart
+      schedule_tasks
+    end
+  end
+end

data/lib/aikido/zen/detached_agent/front_object.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# dRB Front object that will work as a bridge communication between child & parent
+# processes.
+# Every method is called from the child but it runs in the parent process.
+module Aikido::Zen::DetachedAgent
+  class FrontObject
+    def initialize(
+      config: Aikido::Zen.config,
+      collector: Aikido::Zen.collector,
+      runtime_settings: Aikido::Zen.runtime_settings,
+      rate_limiter: Aikido::Zen::RateLimiter.new
+    )
+      @config = config
+      @collector = collector
+      @rate_limiter = rate_limiter
+      @runtime_settings = runtime_settings
+    end
+
+    RequestKind = Struct.new(:route, :schema, :ip, :actor)
+
+    def send_heartbeat_to_parent_process(heartbeat)
+      @collector.push_heartbeat(heartbeat)
+    end
+
+    # Method called by child processes to get an up-to-date version of the
+    # runtime_settings
+    def updated_settings
+      @runtime_settings
+    end
+
+    def calculate_rate_limits(route, ip, actor_hash)
+      actor = Aikido::Zen::Actor(actor_hash) if actor_hash
+      @rate_limiter.calculate_rate_limits(RequestKind.new(route, nil, ip, actor))
+    end
+  end
+end

data/lib/aikido/zen/detached_agent/server.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Aikido::Zen::DetachedAgent
+  class Server
+    def initialize(config: Aikido::Zen.config)
+      @detached_agent_front = FrontObject.new
+      @drb_server = DRb.start_service(config.detached_agent_socket_path, @detached_agent_front)
+
+      # We don't want to see drb logs unless in debug mode
+      @drb_server.verbose = config.logger.debug?
+    end
+
+    def alive?
+      @drb_server.alive?
+    end
+
+    def stop!
+      @drb_server.stop_service
+      DRb.stop_service
+    end
+
+    class << self
+      def start!
+        Aikido::Zen.config.logger.debug("Starting DRb Server...")
+        max_attempts = 10
+        @server = new
+
+        attempts = 0
+        until @server.alive?
+          Aikido::Zen.config.logger.info("DRb Server still not alive. #{max_attempts - attempts} attempts remaining")
+          sleep 0.1
+          attempts += 1
+          raise Aikido::Zen::DetachedAgentError.new("Impossible to start the dRB server (socket=#{Aikido::Zen.config.detached_agent_socket_path})") \
+            if attempts == max_attempts
+        end
+
+        @server
+      end
+    end
+  end
+end

data/lib/aikido/zen/detached_agent.rb

@@ -0,0 +1,2 @@
+require_relative "detached_agent/agent"
+require_relative "detached_agent/server"

data/lib/aikido/zen/errors.rb

@@ -60,7 +60,6 @@ module Aikido
       attr_reader :attack
 
       def initialize(attack)
-        super(attack.log_message)
         @attack = attack
       end
     end
@@ -75,6 +74,16 @@ module Aikido
       def_delegators :@attack, :request, :input
     end
 
+    class PathTraversalError < UnderAttackError
+      extend Forwardable
+      def_delegators :@attack, :input
+    end
+
+    class ShellInjectionError < UnderAttackError
+      extend Forwardable
+      def_delegators :@attack, :input
+    end
+
     # Raised when there's any problem communicating (or loading) libzen.
     class InternalsError < ZenError
       # @param attempt [String] description of what we were trying to do.
@@ -86,5 +95,13 @@ module Aikido
         MSG
       end
     end
+
+    class DetachedAgentError < ZenError
+      extend Forwardable
+
+      def initialize(msg)
+        super
+      end
+    end
   end
 end

data/lib/aikido/zen/event.rb

@@ -48,12 +48,13 @@ module Aikido::Zen
     end
 
     class Heartbeat < Event
-      def initialize(stats:, users:, hosts:, routes:, **opts)
+      def initialize(stats:, users:, hosts:, routes:, middleware_installed:, **opts)
         super(type: "heartbeat", **opts)
         @stats = stats
         @users = users
         @hosts = hosts
         @routes = routes
+        @middleware_installed = middleware_installed
       end
 
       def as_json
@@ -61,7 +62,8 @@ module Aikido::Zen
           stats: @stats.as_json,
           users: @users.as_json,
           routes: @routes.as_json,
-          hostnames: @hosts.as_json
+          hostnames: @hosts.as_json,
+          middlewareInstalled: @middleware_installed
         )
       end
     end

data/lib/aikido/zen/middleware/check_allowed_addresses.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "../context"
-
 module Aikido::Zen
   module Middleware
     # Middleware that rejects requests from IPs blocked in the Aikido dashboard.
@@ -13,24 +11,14 @@ module Aikido::Zen
       end
 
       def call(env)
-        request = request_from(env)
+        request = Aikido::Zen::Middleware.request_from(env)
 
         allowed_ips = @settings.endpoints[request.route].allowed_ips
 
         if allowed_ips.empty? || allowed_ips.include?(request.ip)
           @app.call(env)
         else
-          @config.blocked_ip_responder.call(request)
-        end
-      end
-
-      private
-
-      def request_from(env)
-        if (current_context = Aikido::Zen.current_context)
-          current_context.request
-        else
-          Context.from_rack_env(env).request
+          @config.blocked_responder.call(request, :ip)
         end
       end
     end

data/lib/aikido/zen/middleware/middleware.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Aikido::Zen::Middleware
+  def self.request_from(env)
+    if (current_context = Aikido::Zen.current_context)
+      current_context.request
+    else
+      Aikido::Zen::Context.from_rack_env(env).request
+    end
+  end
+end

data/lib/aikido/zen/middleware/{throttler.rb → rack_throttler.rb}

@@ -4,24 +4,24 @@ require_relative "../context"
 
 module Aikido::Zen
   module Middleware
-    # Middleware that rejects requests from clients that are making too many
+    # Rack middleware that rejects requests from clients that are making too many
     # requests to a given endpoint, based in the runtime configuration in the
     # Aikido dashboard.
-    class Throttler
+    class RackThrottler
       def initialize(
         app,
         config: Aikido::Zen.config,
         settings: Aikido::Zen.runtime_settings,
-        rate_limiter: Aikido::Zen::RateLimiter.new
+        detached_agent: Aikido::Zen.detached_agent
       )
         @app = app
         @config = config
         @settings = settings
-        @rate_limiter = rate_limiter
+        @detached_agent = detached_agent
       end
 
       def call(env)
-        request = request_from(env)
+        request = Aikido::Zen::Middleware.request_from(env)
 
         if should_throttle?(request)
           @config.rate_limited_responder.call(request)
@@ -33,17 +33,15 @@ module Aikido::Zen
       private
 
       def should_throttle?(request)
+        return false unless @settings.endpoints[request.route].rate_limiting.enabled?
         return false if @settings.skip_protection_for_ips.include?(request.ip)
 
-        @rate_limiter.throttle?(request)
-      end
+        result = @detached_agent.calculate_rate_limits(request)
 
-      def request_from(env)
-        if (current_context = Aikido::Zen.current_context)
-          current_context.request
-        else
-          Context.from_rack_env(env).request
-        end
+        return false unless result
+
+        request.env["aikido.rate_limiting"] = result
+        request.env["aikido.rate_limiting"].throttled?
       end
     end
   end

data/lib/aikido/zen/middleware/request_tracker.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+module Aikido::Zen
+  module Middleware
+    # Rack middleware used to track request
+    # It implements the logic under that which is considered worthy of being tracked.
+    class RequestTracker
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        request = Aikido::Zen::Middleware.request_from(env)
+        response = @app.call(env)
+
+        Aikido::Zen.track_request request
+
+        if Aikido::Zen.config.collect_api_schema? && request.route && track?(
+          status_code: response[0],
+          route: request.route.path,
+          http_method: request.request_method
+        )
+          Aikido::Zen.track_discovered_route(request)
+        end
+
+        response
+      end
+
+      IGNORED_METHODS = %w[OPTIONS HEAD]
+      IGNORED_EXTENSIONS = %w[properties config webmanifest]
+      IGNORED_SEGMENTS = ["cgi-bin"]
+      WELL_KNOWN_URIS = %w[
+        /.well-known/acme-challenge
+        /.well-known/amphtml
+        /.well-known/api-catalog
+        /.well-known/appspecific
+        /.well-known/ashrae
+        /.well-known/assetlinks.json
+        /.well-known/broadband-labels
+        /.well-known/brski
+        /.well-known/caldav
+        /.well-known/carddav
+        /.well-known/change-password
+        /.well-known/cmp
+        /.well-known/coap
+        /.well-known/coap-eap
+        /.well-known/core
+        /.well-known/csaf
+        /.well-known/csaf-aggregator
+        /.well-known/csvm
+        /.well-known/did.json
+        /.well-known/did-configuration.json
+        /.well-known/dnt
+        /.well-known/dnt-policy.txt
+        /.well-known/dots
+        /.well-known/ecips
+        /.well-known/edhoc
+        /.well-known/enterprise-network-security
+        /.well-known/enterprise-transport-security
+        /.well-known/est
+        /.well-known/genid
+        /.well-known/gnap-as-rs
+        /.well-known/gpc.json
+        /.well-known/gs1resolver
+        /.well-known/hoba
+        /.well-known/host-meta
+        /.well-known/host-meta.json
+        /.well-known/hosting-provider
+        /.well-known/http-opportunistic
+        /.well-known/idp-proxy
+        /.well-known/jmap
+        /.well-known/keybase.txt
+        /.well-known/knx
+        /.well-known/looking-glass
+        /.well-known/masque
+        /.well-known/matrix
+        /.well-known/mercure
+        /.well-known/mta-sts.txt
+        /.well-known/mud
+        /.well-known/nfv-oauth-server-configuration
+        /.well-known/ni
+        /.well-known/nodeinfo
+        /.well-known/nostr.json
+        /.well-known/oauth-authorization-server
+        /.well-known/oauth-protected-resource
+        /.well-known/ohttp-gateway
+        /.well-known/openid-federation
+        /.well-known/open-resource-discovery
+        /.well-known/openid-configuration
+        /.well-known/openorg
+        /.well-known/oslc
+        /.well-known/pki-validation
+        /.well-known/posh
+        /.well-known/privacy-sandbox-attestations.json
+        /.well-known/private-token-issuer-directory
+        /.well-known/probing.txt
+        /.well-known/pvd
+        /.well-known/rd
+        /.well-known/related-website-set.json
+        /.well-known/reload-config
+        /.well-known/repute-template
+        /.well-known/resourcesync
+        /.well-known/sbom
+        /.well-known/security.txt
+        /.well-known/ssf-configuration
+        /.well-known/sshfp
+        /.well-known/stun-key
+        /.well-known/terraform.json
+        /.well-known/thread
+        /.well-known/time
+        /.well-known/timezone
+        /.well-known/tdmrep.json
+        /.well-known/tor-relay
+        /.well-known/tpcd
+        /.well-known/traffic-advice
+        /.well-known/trust.txt
+        /.well-known/uma2-configuration
+        /.well-known/void
+        /.well-known/webfinger
+        /.well-known/webweaver.json
+        /.well-known/wot
+      ]
+
+      # @param status_code [Integer]
+      # @param route [String]
+      # @param http_method [String]
+      def track?(status_code:, route:, http_method:)
+        # In the UI we want to show only successful (2xx) or redirect (3xx) responses
+        # anything else is discarded.
+        return false unless status_code >= 200 && status_code <= 399
+
+        return false if IGNORED_METHODS.include?(http_method)
+
+        segments = route.split "/"
+
+        # Do not discover routes with dot files like `/path/to/.file` or `/.directory/file`
+        # We want to allow discovery of well-known URIs like `/.well-known/acme-challenge`
+        return false if segments.any? { |s| is_dot_file s } && !is_well_known_uri(route)
+
+        return false if segments.any? { |s| contains_ignored_string s }
+
+        # Check for every file segment if it contains a file extension and if it
+        # should be discovered or ignored
+        segments.all? { |s| should_track_extension s }
+      end
+
+      private
+
+      # Check if a path is a well-known URI
+      # e.g. /.well-known/acme-challenge
+      # https://www.iana.org/assignments/well-known-uris/well-known-uris.xhtml
+      def is_well_known_uri(route)
+        WELL_KNOWN_URIS.include?(route)
+      end
+
+      def is_dot_file(segment)
+        segment.start_with?(".") && segment.size > 1
+      end
+
+      def contains_ignored_string(segment)
+        IGNORED_SEGMENTS.any? { |ignored| segment.include?(ignored) }
+      end
+
+      # Ignore routes which contain file extensions
+      def should_track_extension(segment)
+        extension = get_file_extension(segment)
+
+        return true unless extension
+
+        # Do not discover files with extensions of 1 to 5 characters,
+        # e.g. file.css, file.js, file.woff2
+        return false if extension.size > 1 && extension.size < 6
+
+        # Ignore some file extensions that are longer than 5 characters or shorter than 2 chars
+        return false if IGNORED_EXTENSIONS.include?(extension)
+
+        true
+      end
+
+      def get_file_extension(segment)
+        extension = File.extname(segment)
+        if extension&.start_with?(".")
+          # Remove the dot from the extension
+          return extension[1..]
+        end
+        extension
+      end
+    end
+  end
+end

data/lib/aikido/zen/middleware/set_context.rb

@@ -15,10 +15,7 @@ module Aikido::Zen
       end
 
       def call(env)
-        context = Context.from_rack_env(env)
-
-        Aikido::Zen.current_context = env[ENV_KEY] = context
-        Aikido::Zen.track_request(context.request)
+        Aikido::Zen.current_context = env[ENV_KEY] = Context.from_rack_env(env)
 
         @app.call(env)
       ensure

data/lib/aikido/zen/outbound_connection_monitor.rb

@@ -5,6 +5,10 @@ module Aikido::Zen
   # any Sink that wraps an HTTP library, and lets us keep track of any hosts to
   # which the app communicates over HTTP.
   module OutboundConnectionMonitor
+    def self.skips_on_nil_context?
+      false
+    end
+
     # This simply reports the connection to the Agent, and always returns +nil+
     # as it's not scanning for any particular attack.
     #

data/lib/aikido/zen/payload.rb

@@ -12,6 +12,8 @@ module Aikido::Zen
       @path = path
     end
 
+    UNKNOWN_PAYLOAD = Payload.new("unknown", "unknown", "unknown")
+
     alias_method :to_s, :value
 
     def ==(other)

data/lib/aikido/zen/rails_engine.rb

@@ -14,6 +14,9 @@ module Aikido::Zen
 
       app.middleware.use Aikido::Zen::Middleware::SetContext
       app.middleware.use Aikido::Zen::Middleware::CheckAllowedAddresses
+      # Request Tracker stats do not consider failed request or 40x, so the middleware
+      # must be the last one wrapping the request.
+      app.middleware.use Aikido::Zen::Middleware::RequestTracker
 
       ActiveSupport.on_load(:action_controller) do
         # Due to how Rails sets up its middleware chain, the routing is evaluated
@@ -57,6 +60,15 @@ module Aikido::Zen
       # that any gems required after aikido-zen are detected and patched
       # accordingly.
       Aikido::Zen.load_sinks!
+
+      # It's important we start after loading sinks, so we can report the installed packages
+      Aikido::Zen.start!
+      Aikido::Zen.start!
+
+      # Agent's bootstrap process has finished —Controllers are patched to block
+      # unwanted requests, sinks are loaded, scanners are running—, so we mark
+      # the agent as installed.
+      Aikido::Zen.middleware_installed!
     end
   end
 end

data/lib/aikido/zen/rate_limiter/breaker.rb

@@ -31,13 +31,13 @@ module Aikido::Zen
       @opened_at = @clock.call
     end
 
-    # @param event [#type] an event which we'll discriminate by type to decide
+    # @param event_type [String] an event type which we'll use to decide
     #   if we should throttle it.
     # @return [Boolean]
-    def throttle?(event)
+    def throttle?(event_type)
       return true if open? && !try_close
 
-      result = @bucket.increment(event.type)
+      result = @bucket.increment(event_type)
       result.throttled?
     end
 

data/lib/aikido/zen/rate_limiter.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "synchronizable"
-require_relative "middleware/throttler"
+require_relative "middleware/rack_throttler"
 
 module Aikido::Zen
   # Keeps track of all requests in this process, broken up by Route and further
@@ -24,23 +24,18 @@ module Aikido::Zen
       }
     end
 
-    # Checks whether the request requires rate limiting. As a side effect, this
-    # will annotate the request with the "aikido.rate_limiting" ENV key, holding
-    # the result of the check, and including useful stats in case you want to
-    # return RateLimit headers..
+    # Calculate based on the configuration whether a request will be
+    # rate-limited or not.
     #
     # @param request [Aikido::Zen::Request]
-    # @return [Boolean]
-    #
-    # @see Aikido::Zen::RateLimiter::Result
-    def throttle?(request)
+    # @return [Aikido::Zen::RateLimiter::Result, nil]
+    def calculate_rate_limits(request)
       settings = settings_for(request.route)
-      return false unless settings.enabled?
+      return nil unless settings.enabled?
 
       bucket = @buckets[request.route]
       key = @config.rate_limiting_discriminator.call(request)
-      request.env["aikido.rate_limiting"] = bucket.increment(key)
-      request.env["aikido.rate_limiting"].throttled?
+      bucket.increment(key)
     end
 
     private

data/lib/aikido/zen/request/rails_router.rb

@@ -58,27 +58,15 @@ module Aikido::Zen
     end
 
     private def build_route(route, request, prefix: request.script_name)
-      Rails::Route.new(route, prefix: prefix, verb: request.request_method)
-    end
-  end
-
-  module Rails
-    class Route < Aikido::Zen::Route
-      attr_reader :verb
+      route_wrapper = ActionDispatch::Routing::RouteWrapper.new(route)
 
-      def initialize(rails_route, verb: rails_route.verb, prefix: nil)
-        @route = ActionDispatch::Routing::RouteWrapper.new(rails_route)
-        @verb = verb
-        @prefix = prefix
+      path = if prefix.present?
+        File.join(prefix.to_s, route_wrapper.path).chomp("/")
+      else
+        route_wrapper.path
       end
 
-      def path
-        if @prefix.present?
-          File.join(@prefix.to_s, @route.path).chomp("/")
-        else
-          @route.path
-        end
-      end
+      Aikido::Zen::Route.new(verb: request.request_method, path: path)
     end
   end
 end