brittle 0.1.0

data/lib/brittle/doctor.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "etc"
+
+module Brittle
+  module Doctor
+    module_function
+
+    def check(output: $stdout)
+      output.puts "ruby: #{RUBY_VERSION}"
+      output.puts "kernel: #{Etc.uname.fetch(:release)}"
+      output.puts "arch: #{Etc.uname.fetch(:machine)}"
+
+      unless RUBY_PLATFORM.include?("linux")
+        output.puts "seccomp: unsupported (Linux only)"
+        return false
+      end
+
+      require "seccomp/notify"
+      features = Seccomp::Notify.features
+      output.puts "features: #{features.map { |name, enabled| "#{name}=#{enabled}" }.join(" ")}"
+      features[:user_notif] && features[:continue]
+    rescue LoadError, StandardError => error
+      output.puts "seccomp: unavailable (#{error.message})"
+      false
+    end
+  end
+end

data/lib/brittle/injection.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Brittle
+  class Injection
+    attr_reader :syscall, :errno, :return_value, :matcher, :port
+
+    def initialize(syscall, errno: nil, return_value: nil, at: nil, every: nil, probability: nil, seed: nil,
+      when_fd_path: nil, when_path: nil, port: nil)
+      raise ArgumentError, "choose exactly one of errno or return_value" if errno.nil? == return_value.nil?
+      unless errno.nil? || errno.is_a?(Integer) || errno.is_a?(Class) && errno < SystemCallError
+        raise ArgumentError, "errno must be an Errno class or integer"
+      end
+      raise ArgumentError, "return_value must be an integer" unless return_value.nil? || return_value.is_a?(Integer)
+      raise ArgumentError, "port must be between 1 and 65535" unless port.nil? || port.is_a?(Integer) && port.between?(1, 65_535)
+
+      @syscall = syscall.to_sym
+      @errno = errno
+      @return_value = return_value
+      @matcher = Matcher.new(at:, every:, probability:, seed:)
+      @when_fd_path = pattern(when_fd_path)
+      @when_path = pattern(when_path)
+      @port = port
+    end
+
+    def candidate?(request, path: nil)
+      return false unless request.syscall == @syscall
+      path ||= request.read_string(request.args[1]) if @when_path && %i[openat newfstatat].include?(request.syscall)
+      return false if @when_path && !@when_path.match?(path.to_s)
+      return false if @when_fd_path && !@when_fd_path.match?(fd_path(request))
+      return false if @port && request.read_sockaddr(request.args[1], request.args[2]).ip_port != @port
+
+      true
+    rescue StandardError
+      false
+    end
+
+    def inject!(request)
+      @errno ? request.error!(@errno) : request.allow!(@return_value)
+    end
+
+    def returned
+      @errno ? errno_name : @return_value
+    end
+
+    def to_h
+      {
+        syscall: @syscall,
+        errno: errno_name,
+        return_value: @return_value,
+        at: matcher.at.empty? ? nil : (matcher.at.length == 1 ? matcher.at.first : matcher.at),
+        every: matcher.every,
+        probability: matcher.probability,
+        seed: matcher.seed,
+        scope: {fd_path: pattern_source(@when_fd_path), path: pattern_source(@when_path), port: @port}.compact,
+        realism: realism
+      }.compact
+    end
+
+    private
+
+    def fd_path(request)
+      File.readlink("/proc/#{request.tid}/fd/#{request.args[0]}")
+    end
+
+    def pattern(value)
+      value && (value.is_a?(Regexp) ? value : Regexp.new(value.to_s))
+    end
+
+    def pattern_source(value) = value&.source
+
+    def errno_name
+      @errno.is_a?(Class) ? @errno.name.delete_prefix("Errno::") : @errno
+    end
+
+    def realism
+      return {realistic: false, note: "EINTR is injected without delivering a signal"} if errno_name == "EINTR"
+      if @return_value
+        return {realistic: false, note: "the emulated return value does not perform the partial syscall side effect"}
+      end
+
+      {realistic: true}
+    end
+  end
+end

data/lib/brittle/journal.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "etc"
+require "fileutils"
+require "json"
+require "rbconfig"
+
+module Brittle
+  class Journal
+    SCHEMA_VERSION = 1
+
+    attr_reader :data
+
+    def initialize(scenario:, harness:, injections:, status:, stdout:, stderr:, timed_out:, arm_state:, artifacts:, command:)
+      @scenario = scenario.to_h
+      @injections = injections
+      @data = {
+        schema_version: SCHEMA_VERSION,
+        scenario: @scenario,
+        environment: {ruby: RUBY_VERSION, kernel: Etc.uname.fetch(:release), arch: Etc.uname.fetch(:machine)},
+        harness: harness,
+        injections: injections,
+        result: result(status, stdout, stderr, timed_out, arm_state, artifacts),
+        reproduce: command
+      }.compact
+    end
+
+    def write(path)
+      FileUtils.mkdir_p(File.dirname(path))
+      File.write(path, JSON.pretty_generate(@data) << "\n")
+      path
+    end
+
+    private
+
+    def result(status, stdout, stderr, timed_out, arm_state, artifacts)
+      expected = stdout.lines.filter_map do |line|
+        match = line.match(/\AOK:([^:\n]+):(\d+)(?::([0-9a-f]{64}))?\s*\z/)
+        [match[1], {size: Integer(match[2]), sha256: match[3]}] if match
+      end.to_h
+      artifacts.each do |name, details|
+        next unless expected.key?(name)
+
+        details[:expected_size] = expected[name][:size]
+        details[:expected_sha256] = expected[name][:sha256] if expected[name][:sha256]
+        details[:truncated] = details[:size] < expected[name][:size]
+        details[:mismatched] = details[:size] != expected[name][:size] ||
+          expected[name][:sha256] && details[:sha256] != expected[name][:sha256]
+      end
+      (expected.keys - artifacts.keys).each { |name| artifacts[name] = {missing: true, expected_size: expected[name][:size]} }
+
+      {
+        exit_status: status&.exitstatus,
+        signal: status&.termsig,
+        stdout: stdout,
+        stderr: stderr,
+        verdict: verdict(status, stdout, timed_out, arm_state, artifacts),
+        fd_leak: arm_state.fd_leak?,
+        fd_counts: {
+          before: arm_state.fd_before,
+          after: arm_state.fd_after,
+          delta: arm_state.fd_before && arm_state.fd_after ? arm_state.fd_after - arm_state.fd_before : nil
+        }.compact,
+        artifacts: artifacts
+      }.compact
+    end
+
+    def verdict(status, stdout, timed_out, arm_state, artifacts)
+      return "hang" if timed_out
+      return "leak" if arm_state.fd_leak?
+      return "swallowed" if artifacts.values.any? { |artifact| artifact[:missing] } || stdout.start_with?("SWALLOWED:")
+      return "corrupt" if artifacts.values.any? { |artifact| artifact[:mismatched] } || stdout.start_with?("CORRUPT:")
+      return "crash" unless status&.success?
+      return "expected" if stdout.start_with?("EXPECTED:")
+
+      error = stdout[/^ERR:((?:\w+::)*\w+):/, 1]
+      return expected_error?(error) ? "expected" : "crash" if error
+      "expected"
+    end
+
+    def expected_error?(error)
+      return false unless error
+
+      errno = error.delete_prefix("Errno::")
+      @injections.any? { |injection| injection[:returned].to_s == errno }
+    end
+  end
+end

data/lib/brittle/marker.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Brittle
+  ARM_PATH = "/tmp/.brittle-arm"
+  DISARM_PATH = "/tmp/.brittle-disarm"
+  MARKER_SYSCALL = :newfstatat
+
+  def self.arm! = File.exist?(ARM_PATH)
+  def self.disarm! = File.exist?(DISARM_PATH)
+end

data/lib/brittle/matcher.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Brittle
+  class Matcher
+    attr_reader :at, :count, :every, :probability, :seed
+
+    def initialize(at: nil, every: nil, probability: nil, seed: nil)
+      selectors = [at, every, probability].compact
+      raise ArgumentError, "choose only one of at, every, or probability" if selectors.length > 1
+
+      @at = Array(at || (selectors.empty? ? 1 : nil))
+      raise ArgumentError, "at must not be empty" if at && @at.empty?
+      validate_positive_integers!(@at, "at") unless @at.empty?
+      validate_positive_integers!([every], "every") if every
+      unless probability.nil? || probability.is_a?(Numeric) && probability.between?(0, 1)
+        raise ArgumentError, "probability must be between 0 and 1"
+      end
+      raise ArgumentError, "seed is required with probability" if probability && seed.nil?
+
+      @every = every
+      @probability = probability
+      @seed = seed
+      @random = Random.new(seed) if probability
+      @count = 0
+    end
+
+    def match?
+      @count += 1
+      return @at.include?(@count) unless @at.empty?
+      return (@count % @every).zero? if @every
+
+      @random.rand < @probability
+    end
+
+    private
+
+    def validate_positive_integers!(values, name)
+      return if values.all? { |value| value.is_a?(Integer) && value.positive? }
+
+      raise ArgumentError, "#{name} must contain positive integers"
+    end
+  end
+end

data/lib/brittle/runner.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "rbconfig"
+require "tmpdir"
+
+module Brittle
+  class Runner
+    def initialize(harness, scenario, timeout: 10, command: nil)
+      raise ArgumentError, "timeout must be positive" unless timeout.is_a?(Numeric) && timeout.positive?
+
+      @harness = File.expand_path(harness)
+      @scenario = scenario
+      @timeout = timeout
+      @command = command
+    end
+
+    def run
+      require "seccomp/notify"
+      raise Error, "seccomp user notification with CONTINUE is unavailable" unless Seccomp::Notify.features[:continue]
+      raise ArgumentError, "harness not found: #{@harness}" unless File.file?(@harness)
+
+      Dir.mktmpdir("brittle-") { |sandbox| supervise(sandbox) }
+    end
+
+    private
+
+    def supervise(sandbox)
+      stdout_reader, stdout_writer = IO.pipe
+      stderr_reader, stderr_writer = IO.pipe
+      syscalls = @scenario.notify_syscalls
+      policy = Seccomp::Notify::Policy.new { notify(*syscalls) }
+      supervisor = Seccomp::Notify.spawn(policy) do
+        stdout_reader.close
+        stderr_reader.close
+        STDOUT.reopen(stdout_writer)
+        STDERR.reopen(stderr_writer)
+        stdout_writer.close
+        stderr_writer.close
+        ENV["BRITTLE_SANDBOX"] = sandbox
+        ports = @scenario.injections.filter_map(&:port).uniq
+        ENV["BRITTLE_PORT"] = ports.first.to_s if ports.one?
+        exec(RbConfig.ruby, "-I", File.expand_path("..", __dir__), @harness)
+      end
+      stdout_writer.close
+      stderr_writer.close
+      stdout_thread = Thread.new { stdout_reader.read }
+      stderr_thread = Thread.new { stderr_reader.read }
+
+      arm_state = ArmState.new
+      injections = []
+      errors = []
+      started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      handler = request_handler(arm_state, injections, started_at)
+      syscalls.each { |syscall| supervisor.on(syscall, &handler) }
+      supervisor.on_error { |error, _request| errors << error.full_message(highlight: false, order: :top) }
+
+      timed_out = false
+      watchdog = Thread.new do
+        sleep(@timeout)
+        timed_out = true
+        Process.kill("KILL", supervisor.target_pid)
+      rescue Errno::ESRCH
+        nil
+      end
+      begin
+        status = supervisor.run
+      ensure
+        watchdog.kill
+        watchdog.join
+      end
+      stderr = stderr_thread.value
+      stderr = "#{stderr}#{errors.join("\n")}" unless errors.empty?
+      Journal.new(
+        scenario: @scenario,
+        harness: @harness,
+        injections:,
+        status:,
+        stdout: stdout_thread.value,
+        stderr:,
+        timed_out:,
+        arm_state:,
+        artifacts: collect_artifacts(sandbox),
+        command: @command
+      )
+    ensure
+      [stdout_reader, stdout_writer, stderr_reader, stderr_writer].compact.each { |io| io.close unless io.closed? }
+    end
+
+    def request_handler(arm_state, injections, started_at)
+      lambda do |request|
+        path = request.read_string(request.args[1]) if request.syscall == MARKER_SYSCALL
+        transition = arm_state.transition(path, fd_count: fd_count(request.tid))
+        if transition || !arm_state.armed?
+          request.continue!(unsafe: true)
+          next
+        end
+
+        injection = @scenario.match(request, path:)
+        unless injection
+          request.continue!(unsafe: true)
+          next
+        end
+
+        injections << {
+          seq: injection.matcher.count,
+          syscall: request.syscall,
+          tid: request.tid,
+          returned: injection.returned,
+          at_ms: ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round
+        }
+        injection.inject!(request)
+      end
+    end
+
+    def fd_count(tid)
+      Dir.children("/proc/#{tid}/fd").length
+    rescue SystemCallError
+      nil
+    end
+
+    def collect_artifacts(sandbox)
+      Dir.glob("#{sandbox}/**/*", File::FNM_DOTMATCH).filter_map do |path|
+        next unless File.file?(path)
+
+        name = path.delete_prefix("#{sandbox}/")
+        [name, {size: File.size(path), sha256: Digest::SHA256.file(path).hexdigest}]
+      end.to_h
+    end
+  end
+end

data/lib/brittle/scenario.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Brittle
+  class Scenario
+    attr_reader :injections
+
+    def initialize(&block)
+      @injections = []
+      instance_eval(&block) if block
+    end
+
+    def inject(syscall, **options)
+      @injections << Injection.new(syscall, **options)
+      self
+    end
+
+    def notify_syscalls
+      (@injections.map(&:syscall) << MARKER_SYSCALL).uniq
+    end
+
+    def match(request, path: nil)
+      @injections.filter_map { |injection| injection if injection.candidate?(request, path:) && injection.matcher.match? }.first
+    end
+
+    def to_h
+      @injections.length == 1 ? @injections.first.to_h : {injections: @injections.map(&:to_h)}
+    end
+  end
+end

data/lib/brittle/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Brittle
+  VERSION = "0.1.0"
+end

data/lib/brittle.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "brittle/version"
+require_relative "brittle/marker"
+require_relative "brittle/matcher"
+require_relative "brittle/injection"
+require_relative "brittle/scenario"
+require_relative "brittle/arm_state"
+require_relative "brittle/journal"
+require_relative "brittle/runner"
+
+module Brittle
+  class Error < StandardError; end
+
+  def self.scenario(&block)
+    Scenario.new(&block)
+  end
+end

data/script/sweep_catalog

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+range=${AT_RANGE:-1..20}
+file_harnesses=(file_write tempfile logger csv_write gzip_writer)
+
+for harness in "${file_harnesses[@]}"; do
+  for errno in ENOSPC EDQUOT; do
+    bundle exec exe/brittle sweep "harness/$harness.rb" --syscall write --errno "$errno" --fd-path /brittle- --at "$range"
+  done
+  for errno in EMFILE ENOENT; do
+    bundle exec exe/brittle sweep "harness/$harness.rb" --syscall openat --errno "$errno" --path /brittle- --at "$range"
+  done
+done
+
+bundle exec exe/brittle sweep harness/gzip_writer.rb --syscall read --errno EIO --at "$range"
+for errno in ECONNREFUSED ETIMEDOUT EINTR; do
+  bundle exec exe/brittle sweep harness/net_http.rb --syscall connect --errno "$errno" --at "$range"
+done
+bundle exec exe/brittle sweep harness/syswrite_loop.rb --syscall write --return-value 10 --at "$range"

data/spike/benchmark.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "seccomp/notify"
+
+iterations = Integer(ENV.fetch("ITERATIONS", "10000"))
+measure = lambda do |&block|
+  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  block.call
+  Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+end
+
+baseline = measure.call do
+  File.open("/dev/null", "w") { |file| iterations.times { file.syswrite("x") } }
+end
+
+policy = Seccomp::Notify::Policy.new { notify :write }
+notified = measure.call do
+  supervisor = Seccomp::Notify.spawn(policy) do
+    File.open("/dev/null", "w") { |file| iterations.times { file.syswrite("x") } }
+  end
+  supervisor.on(:write) { |request| request.continue! }
+  supervisor.run
+end
+
+puts "iterations=#{iterations}"
+puts "baseline_ms=#{(baseline * 1000).round(2)}"
+puts "notified_ms=#{(notified * 1000).round(2)}"
+puts "overhead_us_per_write=#{((notified - baseline) * 1_000_000 / iterations).round(2)}"

data/spike/enospc.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "seccomp/notify"
+require "tmpdir"
+
+def inject(response)
+  reader, writer = IO.pipe
+  policy = Seccomp::Notify::Policy.new { notify :write }
+  Dir.mktmpdir("brittle-spike-") do |directory|
+    path = File.join(directory, "out.txt")
+    supervisor = Seccomp::Notify.spawn(policy) do
+      reader.close
+      result = begin
+        written = File.write(path, "x" * 100)
+        "OK:reported=#{written}:actual=#{File.size(path)}"
+      rescue => error
+        "ERR:#{error.class}:#{error.message}"
+      end
+      writer.write(result)
+      writer.close
+    end
+    writer.close
+    count = 0
+    supervisor.on(:write) do |request|
+      count += 1
+      count == 1 ? response.call(request) : request.continue!
+    end
+    supervisor.run
+    reader.read
+  end
+ensure
+  reader&.close
+end
+
+puts inject(->(request) { request.error!(Errno::ENOSPC) })
+puts inject(->(request) { request.allow!(10) })

data/verify/README.md

@@ -0,0 +1,10 @@
+# Real-condition verification
+
+Fault injection finds candidates; these checks reproduce them without seccomp before they are
+reported upstream.
+
+```bash
+docker run --rm --tmpfs /full:rw,size=64k \
+  -v "$PWD:/app" -w /app brittle-dev \
+  bundle exec ruby verify/logger_enospc.rb /full
+```

data/verify/logger_enospc.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "logger"
+
+directory = ARGV.fetch(0) { abort "usage: ruby verify/logger_enospc.rb FULL_FILESYSTEM" }
+log_path = File.join(directory, "app.log")
+fill_path = File.join(directory, "fill")
+
+begin
+  File.open(fill_path, "wb") { |file| loop { file.syswrite("x" * 4096) } }
+  abort "failed to fill filesystem"
+rescue Errno::ENOSPC
+  attempts = 5
+  before = Dir.children("/proc/self/fd").length
+  failures = attempts.times.count do
+    Logger.new(log_path)
+    false
+  rescue Errno::ENOSPC
+    File.unlink(log_path) if File.exist?(log_path)
+    true
+  end
+  after = Dir.children("/proc/self/fd").length
+
+  puts "logger=#{Logger::VERSION} failures=#{failures} fd_before=#{before} fd_after=#{after} delta=#{after - before}"
+  abort "expected one leaked fd per failed initialization" unless failures == attempts && after - before == attempts
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: brittle
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Yudai Takada
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: seccomp-notify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+description: Brittle injects errno responses into selected Linux syscalls using seccomp
+  user notifications.
+email:
+- t.yudai92@gmail.com
+executables:
+- brittle
+extensions: []
+extra_rdoc_files: []
+files:
+- ".dockerignore"
+- Dockerfile.dev
+- LICENSE.txt
+- NOTES.md
+- README.md
+- Rakefile
+- exe/brittle
+- findings/logger-enospc-fd-leak.md
+- harness/csv_write.rb
+- harness/file_write.rb
+- harness/gzip_writer.rb
+- harness/logger.rb
+- harness/net_http.rb
+- harness/syswrite_loop.rb
+- harness/tempfile.rb
+- lib/brittle.rb
+- lib/brittle/arm_state.rb
+- lib/brittle/cli.rb
+- lib/brittle/doctor.rb
+- lib/brittle/injection.rb
+- lib/brittle/journal.rb
+- lib/brittle/marker.rb
+- lib/brittle/matcher.rb
+- lib/brittle/runner.rb
+- lib/brittle/scenario.rb
+- lib/brittle/version.rb
+- script/sweep_catalog
+- spike/benchmark.rb
+- spike/enospc.rb
+- verify/README.md
+- verify/logger_enospc.rb
+homepage: https://github.com/ydah/brittle
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  source_code_uri: https://github.com/ydah/brittle
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.19
+specification_version: 4
+summary: Deterministic syscall fault injection for Ruby programs
+test_files: []