aikido-zen 0.1.1-x86_64-darwin → 1.0.0.pre.beta.1-x86_64-darwin

data/docs/config.md

@@ -17,7 +17,7 @@ requiring a full deploy.)
 ## Blocking mode
 
 In order to have Aikido block requests that look like attacks, you can set
-`AIKIDO_BLOCKING=true` in your environment, or set
+`AIKIDO_BLOCK=true` in your environment, or set
 `Aikido::Zen.config.blocking_mode = true`.
 
 (We recommend the ENV variable as you can normally change this easily without
@@ -79,21 +79,23 @@ Aikido::Zen.rate_limited_responder = ->(request) {
 }
 ```
 
-## IP Blocking responses
+## Blocking responses
 
-If you're using the IP blocking features of Zen, you can configure the response
+If you're using the IP, users or bot blocking features of Zen, you can configure the response
 we send users when their request is rejected with a Proc that returns a
 Rack-compatible response tuple, like this:
 
 ``` ruby
-Aikido::Zen.blocked_ip_responder = ->(request) {
+Aikido::Zen.blocked_responder = ->(request, blocking_type) {
   # Here, request is an instance of Aikido::Zen::Request, which follows the
   # underlying Rack::Request (or ActionDispatch::Request in Rails) API.
-  [403, {"Content-Type" => "application/json"}, ['{"error":"ip_blocked"}']]
+  # And blocking_type is [:ip, :user]
+  
+  [403, {"Content-Type" => "application/json"}, ['{"error":"#{blocking_type.to_s}_blocked"}']]
 }
 ```
 
-By default, Zen emits a `text/plain` 403 response that tells the user their IP
+By default, Zen emits a `text/plain` 403 response that tells the user the request
 is not allowed.
 
 ## API schema sampling

data/docs/rails.md

@@ -48,7 +48,7 @@ Rails.application.config.zen.token = Rails.application.credentials.zen.token
 ## Blocking mode
 
 By default, Zen will only detect and log attacks, but will not block them. You
-can enable blocking mode by setting the `AIKIDO_BLOCKING` environment variable
+can enable blocking mode by setting the `AIKIDO_BLOCK` environment variable
 to `true`.
 
 When in blocking mode, Zen will raise an exception when it detects an attack.

data/lib/aikido/zen/agent.rb

@@ -19,6 +19,7 @@ module Aikido::Zen
     def initialize(
       config: Aikido::Zen.config,
       collector: Aikido::Zen.collector,
+      detached_agent: Aikido::Zen.detached_agent,
       worker: Aikido::Zen::Worker.new(config: config),
       api_client: Aikido::Zen::APIClient.new(config: config)
     )
@@ -28,6 +29,7 @@ module Aikido::Zen
       @worker = worker
       @api_client = api_client
       @collector = collector
+      @detached_agent = detached_agent
     end
 
     def started?
@@ -35,7 +37,7 @@ module Aikido::Zen
     end
 
     def start!
-      @config.logger.info "Starting Aikido agent"
+      @config.logger.info "Starting Aikido agent v#{Aikido::Zen::VERSION}"
 
       raise Aikido::ZenError, "Aikido Agent already started!" if started?
       @started_at = Time.now.utc
@@ -57,7 +59,7 @@ module Aikido::Zen
       at_exit { stop! if started? }
 
       report(Events::Started.new(time: @started_at)) do |response|
-        Aikido::Zen.runtime_settings.update_from_json(response)
+        updated_settings! if Aikido::Zen.runtime_settings.update_from_json(response)
         @config.logger.info "Updated runtime settings."
       rescue => err
         @config.logger.error(err.message)
@@ -103,7 +105,9 @@ module Aikido::Zen
     def handle_attack(attack)
       attack.will_be_blocked! if @config.blocking_mode?
 
-      @config.logger.error("[ATTACK DETECTED] #{attack.log_message}")
+      @config.logger.error(
+        format("Zen has %s a %s: %s", attack.blocked? ? "blocked" : "detected", attack.humanized_name, attack.as_json.to_json)
+      )
       report(Events::Attack.new(attack: attack)) if @api_client.can_make_requests?
 
       @collector.track_attack(attack)
@@ -139,11 +143,11 @@ module Aikido::Zen
     def send_heartbeat(at: Time.now.utc)
       return unless @api_client.can_make_requests?
 
-      event = @collector.flush(at: at)
-
-      report(event) do |response|
-        Aikido::Zen.runtime_settings.update_from_json(response)
-        @config.logger.info "Updated runtime settings after heartbeat"
+      @collector.flush_heartbeats.each do |heartbeat|
+        report(heartbeat) do |response|
+          updated_settings! if Aikido::Zen.runtime_settings.update_from_json(response)
+          @config.logger.info "Updated runtime settings after heartbeat"
+        end
       end
     end
 
@@ -157,7 +161,7 @@ module Aikido::Zen
     def poll_for_setting_updates
       @worker.every(@config.polling_interval) do
         if @api_client.should_fetch_settings?
-          Aikido::Zen.runtime_settings.update_from_json(@api_client.fetch_settings)
+          updated_settings! if Aikido::Zen.runtime_settings.update_from_json(@api_client.fetch_settings)
           @config.logger.info "Updated runtime settings after polling"
         end
       end

data/lib/aikido/zen/api_client.rb

@@ -36,7 +36,7 @@ module Aikido::Zen
 
       response = request(
         Net::HTTP::Get.new("/config", default_headers),
-        base_url: @config.runtime_api_base_url
+        base_url: @config.realtime_endpoint
       )
 
       new_updated_at = Time.at(response["configUpdatedAt"].to_i / 1000)
@@ -72,16 +72,26 @@ module Aikido::Zen
     #   @return (see #fetch_settings)
     #   @raise (see #request)
     def report(event)
-      if @rate_limiter.throttle?(event)
-        @config.logger.error("Not reporting #{event.type.upcase} event due to rate limiting")
+      event_type = if event.respond_to?(:type)
+        event.type
+      else
+        event[:type]
+      end
+
+      if @rate_limiter.throttle?(event_type)
+        @config.logger.error("Not reporting #{event_type.upcase} event due to rate limiting")
         return
       end
 
-      @config.logger.debug("Reporting #{event.type.upcase} event")
+      @config.logger.debug("Reporting #{event_type.upcase} event")
 
       req = Net::HTTP::Post.new("/api/runtime/events", default_headers)
       req.content_type = "application/json"
-      req.body = @config.json_encoder.call(event.as_json)
+      req.body = if event.respond_to?(:as_json)
+        @config.json_encoder.call(event.as_json)
+      else
+        @config.json_encoder.call(event)
+      end
 
       request(req)
     rescue Aikido::Zen::RateLimitedError
@@ -93,14 +103,14 @@ module Aikido::Zen
     # response.
     #
     # @param request [Net::HTTPRequest]
-    # @param base_url [URI] which API to use. Defaults to +Config#api_base_url+.
+    # @param base_url [URI] which API to use. Defaults to +Config#api_endpoint+.
     #
     # @return [Object] the result of decoding the JSON response from the server.
     #
     # @raise [Aikido::Zen::APIError] in case of a 4XX or 5XX response.
     # @raise [Aikido::Zen::NetworkError] if an error occurs trying to make the
     #   request.
-    private def request(request, base_url: @config.api_base_url)
+    private def request(request, base_url: @config.api_endpoint)
       Net::HTTP.start(base_url.host, base_url.port, http_settings) do |http|
         response = http.request(request)
 

data/lib/aikido/zen/attack.rb

@@ -25,20 +25,93 @@ module Aikido::Zen
       @blocked
     end
 
-    def log_message
+    def humanized_name
       raise NotImplementedError, "implement in subclasses"
     end
 
-    def as_json
+    def kind
+      raise NotImplementedError, "implement in subclasses"
+    end
+
+    def input
+      raise NotImplementedError, "implement in subclasses"
+    end
+
+    def metadata
       raise NotImplementedError, "implement in subclasses"
     end
 
+    def as_json
+      {
+        kind: kind,
+        blocked: blocked?,
+        metadata: metadata,
+        operation: @operation
+      }.merge(input.as_json)
+    end
+
     def exception(*)
       raise NotImplementedError, "implement in subclasses"
     end
   end
 
   module Attacks
+    class PathTraversalAttack < Attack
+      attr_reader :input
+      attr_reader :filepath
+
+      def initialize(input:, filepath:, **opts)
+        super(**opts)
+        @input = input
+        @filepath = filepath
+      end
+
+      def metadata
+        {filename: filepath}
+      end
+
+      def humanized_name
+        "path traversal attack"
+      end
+
+      def kind
+        "path_traversal"
+      end
+
+      def exception(*)
+        PathTraversalError.new(self)
+      end
+    end
+
+    class ShellInjectionAttack < Attack
+      attr_reader :input
+      attr_reader :command
+
+      def initialize(input:, command:, **opts)
+        super(**opts)
+        @input = input
+        @command = command
+      end
+
+      def humanized_name
+        "shell injection"
+      end
+
+      def kind
+        "shell_injection"
+      end
+
+      def metadata
+        {
+          command: @command
+        }
+      end
+
+      def exception(*)
+        ShellInjectionError.new(self)
+      end
+    end
+
     class SQLInjectionAttack < Attack
       attr_reader :query
       attr_reader :input
@@ -51,20 +124,16 @@ module Aikido::Zen
         @dialect = dialect
       end
 
-      def log_message
-        format(
-          "SQL Injection: Malicious user input «%s» detected in %s query «%s»",
-          @input, @dialect, @query
-        )
+      def humanized_name
+        "SQL injection"
       end
 
-      def as_json
-        {
-          kind: "sql_injection",
-          blocked: blocked?,
-          metadata: {sql: @query},
-          operation: @operation
-        }.merge(@input.as_json)
+      def kind
+        "sql_injection"
+      end
+
+      def metadata
+        {sql: @query}
       end
 
       def exception(*)
@@ -82,24 +151,23 @@ module Aikido::Zen
         @request = request
       end
 
-      def log_message
-        format(
-          "SSRF: Request to user-supplied hostname «%s» detected in %s (%s).",
-          @input, @operation, @request
-        ).strip
+      def humanized_name
+        "server-side request forgery"
+      end
+
+      def kind
+        "ssrf"
       end
 
       def exception(*)
         SSRFDetectedError.new(self)
       end
 
-      def as_json
+      def metadata
         {
-          kind: "ssrf",
-          metadata: {host: @request.uri.hostname, port: @request.uri.port},
-          blocked: blocked?,
-          operation: @operation
-        }.merge(@input.as_json)
+          host: @request.uri.hostname,
+          port: @request.uri.port
+        }
       end
     end
 
@@ -115,23 +183,24 @@ module Aikido::Zen
         @address = address
       end
 
-      def log_message
-        format(
-          "Stored SSRF: Request to sensitive host «%s» (%s) detected from unknown source in %s",
-          @hostname, @address, @operation
-        )
+      def humanized_name
+        "server-side request forgery"
       end
 
       def exception(*)
         SSRFDetectedError.new(self)
       end
 
-      def as_json
-        {
-          kind: "ssrf",
-          blocked: blocked?,
-          operation: @operation
-        }
+      def kind
+        "ssrf"
+      end
+
+      def input
+        Aikido::Zen::Payload::UNKNOWN_PAYLOAD
+      end
+
+      def metadata
+        {}
       end
     end
   end

data/lib/aikido/zen/background_worker.rb

@@ -0,0 +1,52 @@
+module Aikido::Zen
+  # Generic background worker class backed by queue. Meant to be used by any
+  # background process that needs to do heavy tasks.
+  class BackgroundWorker
+    # @param block [block] A block that receives 1 message directly from the queue
+    def initialize(&block)
+      @queue = Queue.new
+      @block = block
+    end
+
+    # starts the background thread, blocking the thread until a new messages arrives
+    # or the queue is stopped.
+    def start
+      @thread = Thread.new do
+        while running? || actions?
+          action = wait_for_action
+          @block.call(action) unless action.nil?
+        end
+      end
+    end
+
+    def restart
+      stop
+      @queue = Queue.new # re-open the queue
+      start
+    end
+
+    # Drain the queue to do not lose any messages
+    def stop
+      @queue.close # stop accepting messages
+      @thread.join # wait for the queue to be drained
+    end
+
+    def enqueue(scan)
+      @queue.push(scan)
+    end
+
+    private
+
+    def actions?
+      !@queue.empty?
+    end
+
+    def running?
+      !@queue.closed?
+    end
+
+    def wait_for_action
+      @queue.pop(false)
+    end
+  end
+end

data/lib/aikido/zen/collector/routes.rb

@@ -7,6 +7,8 @@ module Aikido::Zen
   #
   # Keeps track of the visited routes.
   class Collector::Routes
+    attr_reader :visits
+
     def initialize(config = Aikido::Zen.config)
       @config = config
       @visits = Hash.new { |h, k| h[k] = Record.new }

data/lib/aikido/zen/collector.rb

@@ -11,6 +11,8 @@ module Aikido::Zen
       @users = Concurrent::AtomicReference.new(Users.new(@config))
       @hosts = Concurrent::AtomicReference.new(Hosts.new(@config))
       @routes = Concurrent::AtomicReference.new(Routes.new(@config))
+      @heartbeats = Queue.new
+      @middleware_installed = Concurrent::AtomicBoolean.new
     end
 
     # Flush all the stats into a Heartbeat event that can be reported back to
@@ -28,7 +30,19 @@ module Aikido::Zen
       start(at: at)
       stats = stats.flush(at: at)
 
-      Events::Heartbeat.new(stats: stats, users: users, hosts: hosts, routes: routes)
+      Events::Heartbeat.new(
+        stats: stats, users: users, hosts: hosts, routes: routes, middleware_installed: middleware_installed?
+      )
+    end
+
+    # Put heartbeats coming from child processes into the internal queue.
+    def push_heartbeat(heartbeat)
+      @heartbeats << heartbeat
+    end
+
+    # Drains into an array all the queued heartbeats
+    def flush_heartbeats
+      Array.new(@heartbeats.size) { @heartbeats.pop }
     end
 
     # Sets the start time for this collection period.
@@ -39,13 +53,17 @@ module Aikido::Zen
       synchronize(@stats) { |stats| stats.start(at) }
     end
 
-    # Track stats about the request, record the visited endpoint, and if
-    # enabled, the API schema for this endpoint.
+    # Track stats about the requests
     #
     # @param request [Aikido::Zen::Request]
     # @return [void]
-    def track_request(request)
+    def track_request(*)
       synchronize(@stats) { |stats| stats.add_request }
+    end
+
+    #  Record the visited endpoint, and if enabled, the API schema for this endpoint.
+    # @param request [Aikido::Zen::Request]
+    def track_route(request)
       synchronize(@routes) { |routes| routes.add(request) if request.route }
     end
 
@@ -83,6 +101,10 @@ module Aikido::Zen
       synchronize(@users) { |users| users.add(actor) }
     end
 
+    def middleware_installed!
+      @middleware_installed.make_true
+    end
+
     # @api private
     def routes
       @routes.get
@@ -103,6 +125,11 @@ module Aikido::Zen
       @stats.get
     end
 
+    # @api private
+    def middleware_installed?
+      @middleware_installed.true?
+    end
+
     # Atomically modify an object's state within a block, ensuring it's safe
     # from other threads.
     private def synchronize(object)

data/lib/aikido/zen/config.rb

@@ -16,18 +16,18 @@ module Aikido::Zen
     alias_method :disabled?, :disabled
 
     # @return [Boolean] whether Aikido should only report infractions or block
-    #   the request by raising an Exception. Defaults to whether AIKIDO_BLOCKING
+    #   the request by raising an Exception. Defaults to whether AIKIDO_BLOCK
     #   is set to a non-empty value in your environment, or +false+ otherwise.
     attr_accessor :blocking_mode
     alias_method :blocking_mode?, :blocking_mode
 
     # @return [URI] The HTTP host for the Aikido API. Defaults to
     #   +https://guard.aikido.dev+.
-    attr_reader :api_base_url
+    attr_reader :api_endpoint
 
     # @return [URI] The HTTP host for the Aikido Runtime API. Defaults to
     #   +https://runtime.aikido.dev+.
-    attr_reader :runtime_api_base_url
+    attr_reader :realtime_endpoint
 
     # @return [Hash] HTTP timeouts for communicating with the API.
     attr_reader :api_timeouts
@@ -53,7 +53,16 @@ module Aikido::Zen
     attr_accessor :json_decoder
 
     # @return [Logger]
-    attr_accessor :logger
+    attr_reader :logger
+
+    # @return [string] Path of the socket where the detached agent will listen.
+    # By default, is stored under the root application path with file name
+    # `aikido-detached-agent.socket`
+    attr_reader :detached_agent_socket_path
+
+    # @return [Boolean] is the agent in debugging mode?
+    attr_accessor :debugging
+    alias_method :debugging?, :debugging
 
     # @return [Integer] maximum number of timing measurements to keep in memory
     #   before compressing them.
@@ -76,10 +85,10 @@ module Aikido::Zen
     #   the oldest seen users.
     attr_accessor :max_users_tracked
 
-    # @return [Proc{Aikido::Zen::Request => Array(Integer, Hash, #each)}]
-    #   Rack handler used to respond to requests from IPs blocked in the Aikido
+    # @return [Proc{(Aikido::Zen::Request, Symbol) => Array(Integer, Hash, #each)}]
+    #   Rack handler used to respond to requests from IPs, users or others blocked in the Aikido
     #   dashboard.
-    attr_accessor :blocked_ip_responder
+    attr_accessor :blocked_responder
 
     # @return [Proc{Aikido::Zen::Request => Array(Integer, Hash, #each)}]
     #   Rack handler used to respond to requests that have been rate limited.
@@ -90,6 +99,12 @@ module Aikido::Zen
     #   differentiate different clients. By default this uses the request IP.
     attr_accessor :rate_limiting_discriminator
 
+    # @return [Boolean] whether Aikido Zen should collect api schemas.
+    #   Defaults to true. Can be set through AIKIDO_FEATURE_COLLECT_API_SCHEMA
+    #   environment variable.
+    attr_accessor :collect_api_schema
+    alias_method :collect_api_schema?, :collect_api_schema
+
     # @return [Integer] max number of requests we sample per endpoint when
     #   computing the schema.
     attr_accessor :api_schema_max_samples
@@ -131,27 +146,30 @@ module Aikido::Zen
 
     def initialize
       self.disabled = read_boolean_from_env(ENV.fetch("AIKIDO_DISABLED", false))
-      self.blocking_mode = read_boolean_from_env(ENV.fetch("AIKIDO_BLOCKING", false))
+      self.blocking_mode = read_boolean_from_env(ENV.fetch("AIKIDO_BLOCK", false))
       self.api_timeouts = 10
-      self.api_base_url = ENV.fetch("AIKIDO_BASE_URL", DEFAULT_API_BASE_URL)
-      self.runtime_api_base_url = ENV.fetch("AIKIDO_RUNTIME_URL", DEFAULT_RUNTIME_BASE_URL)
+      self.api_endpoint = ENV.fetch("AIKIDO_ENDPOINT", DEFAULT_AIKIDO_ENDPOINT)
+      self.realtime_endpoint = ENV.fetch("AIKIDO_REALTIME_ENDPOINT", DEFAULT_RUNTIME_BASE_URL)
       self.api_token = ENV.fetch("AIKIDO_TOKEN", nil)
       self.polling_interval = 60
       self.initial_heartbeat_delay = 60
       self.json_encoder = DEFAULT_JSON_ENCODER
       self.json_decoder = DEFAULT_JSON_DECODER
-      self.logger = Logger.new($stdout, progname: "aikido")
+      self.debugging = read_boolean_from_env(ENV.fetch("AIKIDO_DEBUG", false))
+      self.logger = Logger.new($stdout, progname: "aikido", level: debugging ? Logger::DEBUG : Logger::INFO)
       self.max_performance_samples = 5000
+      self.detached_agent_socket_path = "aikido-detached-agent.socket"
       self.max_compressed_stats = 100
       self.max_outbound_connections = 200
       self.max_users_tracked = 1000
       self.request_builder = Aikido::Zen::Context::RACK_REQUEST_BUILDER
-      self.blocked_ip_responder = DEFAULT_BLOCKED_IP_RESPONDER
+      self.blocked_responder = DEFAULT_BLOCKED_RESPONDER
       self.rate_limited_responder = DEFAULT_RATE_LIMITED_RESPONDER
       self.rate_limiting_discriminator = DEFAULT_RATE_LIMITING_DISCRIMINATOR
       self.server_rate_limit_deadline = 1800 # 30 min
       self.client_rate_limit_period = 3600 # 1 hour
       self.client_rate_limit_max_events = 100
+      self.collect_api_schema = read_boolean_from_env(ENV.fetch("AIKIDO_FEATURE_COLLECT_API_SCHEMA", true))
       self.api_schema_max_samples = Integer(ENV.fetch("AIKIDO_MAX_API_DISCOVERY_SAMPLES", 10))
       self.api_schema_collection_max_depth = 20
       self.api_schema_collection_max_properties = 20
@@ -161,15 +179,22 @@ module Aikido::Zen
     # Set the base URL for API requests.
     #
     # @param url [String, URI]
-    def api_base_url=(url)
-      @api_base_url = URI(url)
+    def api_endpoint=(url)
+      @api_endpoint = URI(url)
     end
 
     # Set the base URL for runtime API requests.
     #
     # @param url [String, URI]
-    def runtime_api_base_url=(url)
-      @runtime_api_base_url = URI(url)
+    def realtime_endpoint=(url)
+      @realtime_endpoint = URI(url)
+    end
+
+    # Set the logger and configure its severity level according to agent's debug mode
+    # @param logger [::Logger]
+    def logger=(logger)
+      @logger = logger
+      @logger.level = Logger::DEBUG if debugging
     end
 
     # @overload def api_timeouts=(timeouts)
@@ -191,6 +216,11 @@ module Aikido::Zen
       @api_timeouts.update(value)
     end
 
+    def detached_agent_socket_path=(path)
+      @detached_agent_socket_path = path
+      @detached_agent_socket_path = "drbunix:" + @detached_agent_socket_path unless @detached_agent_socket_path.start_with?("drbunix:")
+    end
+
     private
 
     def read_boolean_from_env(value)
@@ -205,7 +235,7 @@ module Aikido::Zen
     end
 
     # @!visibility private
-    DEFAULT_API_BASE_URL = "https://guard.aikido.dev"
+    DEFAULT_AIKIDO_ENDPOINT = "https://guard.aikido.dev"
 
     # @!visibility private
     DEFAULT_RUNTIME_BASE_URL = "https://runtime.aikido.dev"
@@ -217,9 +247,14 @@ module Aikido::Zen
     DEFAULT_JSON_DECODER = JSON.method(:parse)
 
     # @!visibility private
-    DEFAULT_BLOCKED_IP_RESPONDER = ->(request) do
-      message = "Your IP address is not allowed to access this resource. (Your IP: %s)"
-      [403, {"Content-Type" => "text/plain"}, [format(message, request.ip)]]
+    DEFAULT_BLOCKED_RESPONDER = ->(request, blocking_type) do
+      message = case blocking_type
+      when :ip
+        format("Your IP address is not allowed to access this resource. (Your IP: %s)", request.ip)
+      else
+        "You are blocked by Zen."
+      end
+      [403, {"Content-Type" => "text/plain"}, [message]]
     end
 
     # @!visibility private