corkscrews 0.1.0

data/lib/corkscrews/controller.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+require_relative "analysis"
+
+module Corkscrews
+  class Controller
+    SPEEDUPS = Analysis::SPEEDUPS.freeze
+    DEFAULT_RANDOM = Random.new(12_345)
+
+    Round = Struct.new(
+      :id,
+      :target,
+      :speedup_pct,
+      :baseline,
+      :visits,
+      :physical_duration_ns,
+      :virtual_duration_ns,
+      keyword_init: true
+    )
+
+    def initialize(targets:, random: DEFAULT_RANDOM)
+      @targets = targets
+      @random = random
+      @round_index = 0
+    end
+
+    def next_round(progress_visits:, duration_ns:, targets: nil, history: [])
+      active_targets = targets || @targets
+      target = pick_target(active_targets, history)
+      speedup_pct = pick_speedup(target, history)
+      share = target_share(target)
+      reduced = share * (speedup_pct.to_f / 100.0)
+
+      Round.new(
+        id: next_round_id,
+        target: target,
+        speedup_pct: speedup_pct,
+        baseline: speedup_pct.zero?,
+        visits: progress_visits,
+        physical_duration_ns: duration_ns,
+        virtual_duration_ns: (duration_ns * (1.0 - reduced)).round
+      )
+    end
+
+    def self.plan_for(analysis, rounds: 32, random: DEFAULT_RANDOM)
+      aggregate = analysis.aggregate
+      controller = new(targets: aggregate[:targets].first(32), random: random)
+      progress_count = aggregate[:progress].values.sum { |entry| entry[:count].to_i }
+      duration_ns = aggregate[:duration_ns]
+
+      Array.new(rounds) do
+        controller.next_round(progress_visits: progress_count, duration_ns: duration_ns)
+      end
+    end
+
+    private
+
+    # Paper basis: Coz SOSP'15 Section 2 "Experiment initialization"
+    # and Section 3.2 select the speedup target/amount during execution;
+    # Stabilizer ASPLOS'13 Section 2 motivates randomized exploration to
+    # avoid systematic measurement bias.
+    # Paper URLs: https://arxiv.org/abs/1608.03676
+    # https://people.cs.umass.edu/~emery/pubs/stabilizer-asplos13.pdf
+    def pick_target(targets, history)
+      return { kind: "none", sample_share: 0.0, causal_share: 0.0 } if targets.empty?
+
+      total = targets.sum { |target| target_weight(target, history) }
+      cursor = @random.rand * total
+      targets.each do |target|
+        cursor -= target_weight(target, history)
+        return target if cursor <= 0.0
+      end
+      targets.last
+    end
+
+    # Paper basis: Coz SOSP'15 Section 3.2 gives 0% virtual speedup
+    # 50% probability because every target needs a local baseline.
+    # Paper URL: https://arxiv.org/abs/1608.03676
+    def pick_speedup(target, history)
+      target_history = history_for_target(target, history)
+      baseline_count = target_history.count { |round| round.fetch(:speedup_pct, round["speedup_pct"]).to_i.zero? }
+      nonbaseline_count = target_history.length - baseline_count
+      return 0 if baseline_count <= nonbaseline_count / 2
+
+      uncovered = priority_speedups.reject do |speedup|
+        target_history.any? { |round| round.fetch(:speedup_pct, round["speedup_pct"]).to_i == speedup }
+      end
+      return uncovered.first if uncovered.any?
+
+      SPEEDUPS.reject(&:zero?).fetch(@random.rand(SPEEDUPS.length - 1))
+    end
+
+    def next_round_id
+      @round_index += 1
+      "round-#{@round_index}-#{SecureRandom.hex(3)}"
+    end
+
+    def target_weight(target, history)
+      count = history_for_target(target, history).length
+      share = [target_share(target), 0.0001].max
+      exploration = 1.0 / Math.sqrt(count + 1)
+      coverage = missing_priority_speedups(target, history) * 0.05
+      share * exploration + coverage
+    end
+
+    def target_share(target)
+      target.fetch(:causal_share, target.fetch(:sample_share, target.fetch(:share, 0.0))).to_f
+    end
+
+    def missing_priority_speedups(target, history)
+      seen = history_for_target(target, history).map { |round| round.fetch(:speedup_pct, round["speedup_pct"]).to_i }
+      priority_speedups.count { |speedup| !seen.include?(speedup) }
+    end
+
+    def priority_speedups
+      [0, 25, 50, 90]
+    end
+
+    def history_for_target(target, history)
+      key = target_key(target)
+      history.select { |round| target_key(round.fetch(:target, round["target"])) == key }
+    end
+
+    def target_key(target)
+      return ["none"] unless target
+
+      kind = target.fetch(:kind, target["kind"]).to_s
+      if kind == "wait"
+        [kind, target.fetch(:name, target["name"]).to_s]
+      else
+        [kind, File.expand_path(target.fetch(:file, target["file"]).to_s), target.fetch(:line, target["line"]).to_i]
+      end
+    end
+  end
+end

data/lib/corkscrews/firefox_profile.rb

@@ -0,0 +1,290 @@
+# frozen_string_literal: true
+
+module Corkscrews
+  class FirefoxProfile
+    MAX_SYNTHETIC_SAMPLES = 10_000
+
+    class << self
+      def build(aggregate)
+        new(aggregate).build
+      end
+    end
+
+    def initialize(aggregate)
+      @aggregate = aggregate
+      @strings = []
+      @string_indexes = {}
+    end
+
+    def build
+      thread = build_thread
+
+      {
+        "meta" => meta,
+        "libs" => [],
+        "pages" => [],
+        "profileGatheringLog" => [],
+        "threads" => [thread]
+      }
+    end
+
+    private
+
+    def meta
+      {
+        "version" => 34,
+        "product" => "corkscrews",
+        "interval" => sample_interval_ms,
+        "processType" => 0,
+        "platform" => RUBY_PLATFORM,
+        "misc" => "Ruby #{RUBY_VERSION}"
+      }
+    end
+
+    def build_thread
+      targets = @aggregate.fetch(:targets, [])
+      funcs = build_funcs(targets)
+      frames = build_frames(targets)
+      stacks = build_stacks(targets)
+      samples = build_samples(targets)
+      markers = build_markers(targets)
+
+      {
+        "name" => "corkscrews aggregate",
+        "processType" => "default",
+        "processName" => "corkscrews",
+        "isMainThread" => true,
+        "pid" => 0,
+        "tid" => 0,
+        "registerTime" => 0,
+        "unregisterTime" => duration_ms,
+        "stringArray" => @strings,
+        "stringTable" => @strings,
+        "resourceTable" => empty_resource_table,
+        "funcTable" => funcs,
+        "frameTable" => frames,
+        "stackTable" => stacks,
+        "samples" => samples,
+        "markers" => markers,
+        "pausedRanges" => []
+      }
+    end
+
+    def build_funcs(targets)
+      {
+        "schema" => {
+          "name" => 0,
+          "isJS" => 1,
+          "resource" => 2,
+          "relevantForJS" => 3
+        },
+        "data" => targets.map { |target| [string_index(target_label(target)), false, -1, false] }
+      }
+    end
+
+    def build_frames(targets)
+      {
+        "schema" => {
+          "address" => 0,
+          "category" => 1,
+          "subcategory" => 2,
+          "func" => 3,
+          "innerWindowID" => 4,
+          "implementation" => 5,
+          "line" => 6,
+          "column" => 7
+        },
+        "data" => targets.each_with_index.map do |target, index|
+          [-1, 0, 0, index, 0, nil, frame_line(target), nil]
+        end
+      }
+    end
+
+    def build_stacks(targets)
+      {
+        "schema" => {
+          "prefix" => 0,
+          "frame" => 1,
+          "category" => 2,
+          "subcategory" => 3
+        },
+        "data" => targets.each_index.map { |index| [nil, index, 0, 0] }
+      }
+    end
+
+    def build_samples(targets)
+      rows = synthetic_sample_rows(targets)
+
+      {
+        "schema" => {
+          "stack" => 0,
+          "time" => 1,
+          "responsiveness" => 2
+        },
+        "data" => rows
+      }
+    end
+
+    def build_markers(targets)
+      {
+        "schema" => {
+          "name" => 0,
+          "startTime" => 1,
+          "endTime" => 2,
+          "phase" => 3,
+          "category" => 4,
+          "data" => 5
+        },
+        "data" => target_markers(targets) + round_markers
+      }
+    end
+
+    def target_markers(targets)
+      targets.map do |target|
+        [
+          string_index("#{target[:verdict]}: #{target_label(target)}"),
+          0,
+          duration_ms,
+          1,
+          0,
+          marker_payload(target)
+        ]
+      end
+    end
+
+    def round_markers
+      cursor_ms = 0.0
+      @aggregate.fetch(:rounds, []).map do |round|
+        duration = round_duration_ms(round)
+        start_time = cursor_ms
+        cursor_ms += duration
+
+        [
+          string_index("round #{round_value(round, :speedup_pct)}% #{target_label(round_target(round))}"),
+          rounded_time(start_time),
+          rounded_time(cursor_ms),
+          1,
+          0,
+          round_marker_payload(round)
+        ]
+      end
+    end
+
+    def synthetic_sample_rows(targets)
+      sample_units = sample_units_for(targets)
+      return [] if sample_units.empty?
+
+      total_rows = sample_units.sum
+      spacing_ms = total_rows.positive? ? duration_ms / total_rows : sample_interval_ms
+      time_ms = 0.0
+
+      sample_units.each_with_index.flat_map do |count, stack_index|
+        Array.new(count) do
+          row = [stack_index, rounded_time(time_ms), 0]
+          time_ms += spacing_ms
+          row
+        end
+      end
+    end
+
+    def sample_units_for(targets)
+      counts = targets.map { |target| target[:samples].to_i }
+      total = counts.sum
+      return [] unless total.positive?
+
+      if total <= MAX_SYNTHETIC_SAMPLES
+        counts
+      else
+        counts.map do |count|
+          count.positive? ? [((count.to_f / total) * MAX_SYNTHETIC_SAMPLES).round, 1].max : 0
+        end
+      end
+    end
+
+    def marker_payload(target)
+      {
+        "type" => "CorkscrewsTarget",
+        "target" => target_label(target),
+        "sampleShare" => target[:sample_share].to_f,
+        "causalShare" => target[:causal_share].to_f,
+        "samples" => target[:samples].to_i,
+        "verdict" => target[:verdict].to_s
+      }
+    end
+
+    def round_marker_payload(round)
+      {
+        "type" => "CorkscrewsRound",
+        "target" => target_label(round_target(round)),
+        "speedupPct" => round_value(round, :speedup_pct).to_i,
+        "baseline" => !!round_value(round, :baseline),
+        "visits" => round_value(round, :visits).to_i,
+        "physicalDurationMs" => round_value(round, :physical_duration_ns).to_f / 1_000_000.0,
+        "virtualDurationMs" => round_value(round, :virtual_duration_ns).to_f / 1_000_000.0
+      }
+    end
+
+    def round_duration_ms(round)
+      duration = round_value(round, :physical_duration_ns).to_f / 1_000_000.0
+      duration.positive? ? duration : sample_interval_ms
+    end
+
+    def round_target(round)
+      round_value(round, :target) || { kind: "none" }
+    end
+
+    def round_value(round, key)
+      round[key] || round[key.to_s]
+    end
+
+    def empty_resource_table
+      {
+        "schema" => {
+          "type" => 0,
+          "name" => 1,
+          "host" => 2
+        },
+        "data" => []
+      }
+    end
+
+    def target_label(target)
+      kind = target[:kind] || target["kind"]
+      if kind == "wait"
+        "wait:#{target[:name] || target["name"]}"
+      elsif kind == "none"
+        "none"
+      else
+        "#{target[:file] || target["file"]}:#{target[:line] || target["line"]}"
+      end
+    end
+
+    def frame_line(target)
+      return nil if (target[:kind] || target["kind"]) == "wait"
+
+      (target[:line] || target["line"]).to_i
+    end
+
+    def sample_interval_ms
+      [duration_ms / [@aggregate.fetch(:total_samples, 0).to_i, 1].max, 0.001].max
+    end
+
+    def duration_ms
+      duration = @aggregate.fetch(:duration_ns, 0).to_f / 1_000_000.0
+      duration.positive? ? duration : 1.0
+    end
+
+    def rounded_time(value)
+      value.round(6)
+    end
+
+    def string_index(value)
+      key = value.to_s
+      @string_indexes.fetch(key) do
+        @string_indexes[key] = @strings.length
+        @strings << key
+        @string_indexes[key]
+      end
+    end
+  end
+end

data/lib/corkscrews/native.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+begin
+  require "corkscrews/corkscrews"
+rescue LoadError
+  begin
+    require_relative "../../ext/corkscrews/corkscrews"
+  rescue LoadError
+    nil
+  end
+end
+
+module Corkscrews
+  module Native
+    @stamps = ObjectSpace::WeakMap.new
+
+    module_function
+
+    def available?
+      defined?(Corkscrews::NativeExtension::AVAILABLE) && Corkscrews::NativeExtension::AVAILABLE
+    end
+
+    def settled_now
+      return Corkscrews::Delay.settled_now if defined?(Corkscrews::Delay)
+
+      Thread.current[:corkscrews_settled_stamp_ns] ||= 0
+    end
+
+    def stamp_write(object)
+      if defined?(Corkscrews::Delay)
+        Corkscrews::Delay.stamp_write(object)
+        return nil
+      end
+
+      @stamps[object] = settled_now
+    end
+
+    def stamp_inherit(object)
+      if defined?(Corkscrews::Delay)
+        Corkscrews::Delay.stamp_inherit(object)
+        return nil
+      end
+
+      stamp = @stamps[object].to_i
+      current = settled_now
+      Thread.current[:corkscrews_settled_stamp_ns] = [current, stamp].max
+    end
+
+    def install_hooks!
+      Corkscrews::Hooks.install! if defined?(Corkscrews::Hooks)
+    end
+
+    def start_monitor
+      Corkscrews::Monitor.start! if defined?(Corkscrews::Monitor)
+    end
+
+    def stop_monitor
+      Corkscrews::Monitor.stop! if defined?(Corkscrews::Monitor)
+    end
+
+    def monitor_available?
+      defined?(Corkscrews::Monitor) &&
+        Corkscrews::Monitor.respond_to?(:available?) &&
+        Corkscrews::Monitor.available?
+    end
+
+    def register_monitor_thread
+      Corkscrews::Monitor.register_current_thread! if defined?(Corkscrews::Monitor) && Corkscrews::Monitor.respond_to?(:register_current_thread!)
+    end
+
+    def clear_monitor_thread
+      Corkscrews::Monitor.clear_current_thread! if defined?(Corkscrews::Monitor) && Corkscrews::Monitor.respond_to?(:clear_current_thread!)
+    end
+
+    def native_snapshot
+      return {} unless defined?(Corkscrews::Record)
+
+      Corkscrews::Record.snapshot
+    end
+
+    def native_events
+      return [] unless defined?(Corkscrews::Record) && Corkscrews::Record.respond_to?(:events)
+
+      Corkscrews::Record.events
+    end
+
+    def set_experiment(kind:, iseq: 0, line: 0, speedup_pct: 0)
+      return unless defined?(Corkscrews::Delay)
+
+      kind_id = { none: 0, line: 1, wait: 2 }.fetch(kind.to_sym, 0)
+      Corkscrews::Delay.set_experiment(kind_id, iseq.to_i, line.to_i, speedup_pct.to_i)
+    end
+
+    def set_sample_period(ns)
+      Corkscrews::Delay.set_sample_period(ns.to_i) if defined?(Corkscrews::Delay)
+    end
+
+    def clear_experiment
+      Corkscrews::Delay.clear_experiment if defined?(Corkscrews::Delay)
+    end
+
+    def settle_current
+      return Corkscrews::Delay.settle_current if defined?(Corkscrews::Delay)
+
+      0
+    end
+  end
+end

data/lib/corkscrews/primitives.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require_relative "native"
+require_relative "time_source"
+
+module Corkscrews
+  module Primitives
+    # Paper basis: Ousterhout et al. NSDI'15 Section 2.3.1 instruments
+    # time spent blocked from the compute thread's perspective; these
+    # prepends adapt that blocked-time accounting to Ruby waits.
+    # Paper URL: https://www.usenix.org/system/files/conference/nsdi15/nsdi15-paper-ousterhout.pdf
+    module MutexStamp
+      def unlock
+        Corkscrews::Native.stamp_write(self)
+        super
+      end
+
+      def lock
+        started_ns = Corkscrews::TimeSource.monotonic_ns
+        result = super
+        Corkscrews.record_wait(:lock_wait, Corkscrews::TimeSource.monotonic_ns - started_ns)
+        Corkscrews::Native.stamp_inherit(self)
+        result
+      end
+
+      def synchronize(&block)
+        lock
+        begin
+          block.call
+        ensure
+          unlock
+        end
+      end
+    end
+
+    module QueueStamp
+      def <<(...)
+        push(...)
+      end
+
+      def push(...)
+        Corkscrews::Native.stamp_write(self)
+        started_ns = Corkscrews::TimeSource.monotonic_ns
+        result = super
+        Corkscrews.record_wait(:queue_wait, Corkscrews::TimeSource.monotonic_ns - started_ns)
+        Corkscrews::Native.stamp_inherit(self)
+        result
+      end
+
+      def pop(...)
+        Corkscrews::Native.stamp_write(self)
+        started_ns = Corkscrews::TimeSource.monotonic_ns
+        result = super
+        Corkscrews.record_wait(:queue_wait, Corkscrews::TimeSource.monotonic_ns - started_ns)
+        Corkscrews::Native.stamp_inherit(self)
+        result
+      end
+
+      def deq(...)
+        pop(...)
+      end
+    end
+
+    module CondVarStamp
+      def signal
+        Corkscrews::Native.stamp_write(self)
+        super
+      end
+
+      def broadcast
+        Corkscrews::Native.stamp_write(self)
+        super
+      end
+
+      def wait(mutex, timeout = nil)
+        started_ns = Corkscrews::TimeSource.monotonic_ns
+        result = super
+        Corkscrews.record_wait(:condition_wait, Corkscrews::TimeSource.monotonic_ns - started_ns)
+        Corkscrews::Native.stamp_inherit(self)
+        result
+      end
+    end
+
+    module KernelWait
+      def sleep(...)
+        started_ns = Corkscrews::TimeSource.monotonic_ns
+        result = super
+        Corkscrews.record_wait(:sleep_wait, Corkscrews::TimeSource.monotonic_ns - started_ns)
+        result
+      end
+    end
+
+    module IOWait
+      def read(...)
+        measure_io_wait { super }
+      end
+
+      def readpartial(...)
+        measure_io_wait { super }
+      end
+
+      def write(...)
+        measure_io_wait { super }
+      end
+
+      private
+
+      def measure_io_wait
+        started_ns = Corkscrews::TimeSource.monotonic_ns
+        result = yield
+        Corkscrews.record_wait(:io_wait, Corkscrews::TimeSource.monotonic_ns - started_ns)
+        result
+      end
+    end
+
+    def self.install!
+      Mutex.prepend MutexStamp unless Mutex.ancestors.include?(MutexStamp)
+      Thread::Queue.prepend QueueStamp unless Thread::Queue.ancestors.include?(QueueStamp)
+      Thread::SizedQueue.prepend QueueStamp unless Thread::SizedQueue.ancestors.include?(QueueStamp)
+      Queue.prepend QueueStamp if defined?(Queue) && !Queue.ancestors.include?(QueueStamp)
+      SizedQueue.prepend QueueStamp if defined?(SizedQueue) && !SizedQueue.ancestors.include?(QueueStamp)
+      Thread::ConditionVariable.prepend CondVarStamp unless Thread::ConditionVariable.ancestors.include?(CondVarStamp)
+      Kernel.prepend KernelWait unless Kernel.ancestors.include?(KernelWait)
+      IO.prepend IOWait unless IO.ancestors.include?(IOWait)
+    end
+  end
+end

data/lib/corkscrews/rack.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "../corkscrews"
+
+module Corkscrews
+  class Rack
+    def initialize(app, progress_name: :rack_request)
+      @app = app
+      @progress_name = progress_name
+    end
+
+    def call(env)
+      Corkscrews.latency_begin(@progress_name)
+      status, headers, body = @app.call(env)
+      Corkscrews.progress(@progress_name)
+      [status, headers, ResponseBody.new(body, @progress_name)]
+    rescue Exception
+      Corkscrews.latency_end(@progress_name)
+      raise
+    end
+
+    class ResponseBody
+      def initialize(body, progress_name)
+        @body = body
+        @progress_name = progress_name
+      end
+
+      def each(&block)
+        @body.each(&block)
+      end
+
+      def close
+        @body.close if @body.respond_to?(:close)
+      ensure
+        Corkscrews.latency_end(@progress_name)
+      end
+    end
+  end
+end