julewire-core 1.0.0

data/lib/julewire/core/scheduling/deadline_scheduler.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Scheduling
+      # @api integration_spi
+      class DeadlineScheduler
+        CLOCK = Process::CLOCK_MONOTONIC
+        Entry = Data.define(:deadline, :token, :callback)
+
+        def initialize(thread_name:, idle: :keep_alive)
+          @thread_name = thread_name
+          @idle = idle
+          @mutex = Mutex.new
+          @condition = ConditionVariable.new
+          @entries = {}
+          # A heap keeps timeout scheduling cheap without non-shareable scheduler dependencies.
+          @heap = []
+          @next_token = 0
+          @generation = 0
+          @pid = Process.pid
+          @thread = nil
+        end
+
+        def schedule(timeout, &block)
+          raise ArgumentError, "block required" unless block
+
+          timeout = Float(timeout)
+          if timeout <= 0
+            yield
+            return
+          end
+
+          @mutex.synchronize do
+            token = next_token
+            entry = Entry.new(monotonic_time + timeout, token, block)
+            @entries[token] = entry
+            heap_push(entry)
+            ensure_thread
+            @condition.signal
+            token
+          end
+        end
+
+        def cancel(token)
+          return unless token
+
+          @mutex.synchronize do
+            @entries.delete(token)
+            @condition.signal
+          end
+        end
+
+        def after_fork!
+          if @pid == Process.pid
+            reset_same_process
+          else
+            reset_after_fork
+          end
+
+          self
+        end
+
+        private
+
+        def reset_same_process
+          @mutex.synchronize do
+            @generation += 1
+            @entries = {}
+            @heap = []
+            @next_token = 0
+            @thread = nil
+            @condition.broadcast
+          end
+        end
+
+        def reset_after_fork
+          @mutex = Mutex.new
+          @condition = ConditionVariable.new
+          @entries = {}
+          @heap = []
+          @next_token = 0
+          @generation += 1
+          @pid = Process.pid
+          @thread = nil
+        end
+
+        def next_token
+          @next_token += 1
+        end
+
+        def ensure_thread
+          return if @thread&.alive?
+
+          generation = @generation
+          @thread = Thread.new { run(generation) }
+          @thread.name = @thread_name
+          @thread.report_on_exception = false
+        end
+
+        def run(generation)
+          loop do
+            callback = next_expired_callback(generation)
+            return unless callback
+
+            safe_call(callback)
+          end
+        end
+
+        def next_expired_callback(generation)
+          @mutex.synchronize do
+            loop do
+              return unless generation == @generation
+
+              discard_cancelled_head
+              if @heap.empty?
+                return clear_thread if exit_when_idle?
+
+                @condition.wait(@mutex)
+                next
+              end
+
+              entry = @heap.fetch(0)
+              remaining = entry.deadline - monotonic_time
+              if remaining.positive?
+                @condition.wait(@mutex, remaining)
+              else
+                heap_pop
+                @entries.delete(entry.token)
+                return entry.callback
+              end
+            end
+          end
+        end
+
+        def discard_cancelled_head
+          heap_pop while @heap.any? && !@entries.key?(@heap.fetch(0).token)
+        end
+
+        def clear_thread
+          @thread = nil
+          nil
+        end
+
+        def exit_when_idle?
+          @idle == :exit
+        end
+
+        def safe_call(callback)
+          callback.call
+        rescue StandardError
+          nil
+        end
+
+        def monotonic_time
+          Process.clock_gettime(CLOCK)
+        end
+
+        def heap_push(entry)
+          @heap << entry
+          sift_up(@heap.length - 1)
+        end
+
+        def heap_pop
+          return @heap.pop if @heap.one?
+
+          top = @heap.fetch(0)
+          @heap[0] = @heap.pop
+          sift_down(0)
+          top
+        end
+
+        def sift_up(index)
+          while index.positive?
+            parent = (index - 1) / 2
+            break if earlier_or_equal?(@heap.fetch(parent), @heap.fetch(index))
+
+            swap_heap(index, parent)
+            index = parent
+          end
+        end
+
+        def sift_down(index)
+          loop do
+            left = (index * 2) + 1
+            right = left + 1
+            smallest = index
+            smallest = left if left < @heap.length && earlier_or_equal?(@heap.fetch(left), @heap.fetch(smallest))
+            smallest = right if right < @heap.length && earlier_or_equal?(@heap.fetch(right), @heap.fetch(smallest))
+            break if smallest == index
+
+            swap_heap(index, smallest)
+            index = smallest
+          end
+        end
+
+        def earlier_or_equal?(left, right)
+          left.deadline < right.deadline || (left.deadline == right.deadline && left.token <= right.token)
+        end
+
+        def swap_heap(left, right)
+          @heap[left], @heap[right] = @heap[right], @heap[left]
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/scheduling/shared_scheduler.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Scheduling
+      module SharedScheduler
+        THREAD_NAME = "julewire-deadline-scheduler"
+
+        @mutex = Mutex.new
+
+        class << self
+          def schedule(timeout, &)
+            current = scheduler
+            current.schedule(timeout, &)
+          end
+
+          def cancel(token)
+            current = scheduler
+            current.cancel(token)
+          end
+
+          def after_fork!
+            current = @scheduler
+            @mutex = Mutex.new
+            current&.after_fork!
+            nil
+          end
+
+          # Private testing seam for isolating process-wide scheduler state.
+          def reset_for_test!
+            @mutex.synchronize { @scheduler = nil }
+            nil
+          end
+
+          private
+
+          private :reset_for_test!
+
+          def scheduler
+            @mutex.synchronize do
+              @scheduler ||= DeadlineScheduler.new(thread_name: THREAD_NAME)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/sentinel.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    class Sentinel
+      attr_reader :name
+
+      def initialize(name)
+        @name = Core.normalize_name(name, name: :sentinel)
+        freeze
+      end
+
+      def inspect = "#<#{self.class} #{@name}>"
+
+      def to_s = inspect
+    end
+  end
+end

data/lib/julewire/core/serialization/backtrace_limiter.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Serialization
+      class BacktraceLimiter
+        class << self
+          def call(value, max_backtrace_lines:)
+            new(max_backtrace_lines: max_backtrace_lines).call(value)
+          end
+        end
+
+        def initialize(max_backtrace_lines:)
+          @max_backtrace_lines = Validation.validate_integer_limit!(
+            max_backtrace_lines,
+            name: :max_backtrace_lines
+          )
+        end
+
+        def call(value)
+          @seen = {}.compare_by_identity
+          limit_backtraces(value)
+          value
+        ensure
+          @seen = nil
+        end
+
+        private
+
+        def limit_backtraces(value)
+          while value.is_a?(Hash) && !@seen.key?(value)
+            @seen[value] = true
+            limit_backtrace_field!(value)
+            value = value[:cause]
+          end
+        end
+
+        def limit_backtrace_field!(error)
+          return unless error.key?(:backtrace)
+
+          if @max_backtrace_lines.zero?
+            error.delete(:backtrace)
+          elsif error[:backtrace].is_a?(Array)
+            error[:backtrace] = error[:backtrace].first(@max_backtrace_lines)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/serialization/bounded_transform.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Serialization
+      # @api integration_spi
+      class BoundedTransform < BoundedTraversal
+        CONTINUE = Core.sentinel(:continue)
+
+        class << self
+          def call(value, **, &)
+            new(**, &).call(value)
+          end
+        end
+
+        def initialize(
+          max_depth: DEFAULT_MAX_DEPTH,
+          max_string_bytes: DEFAULT_MAX_STRING_BYTES,
+          max_array_items: DEFAULT_MAX_ARRAY_ITEMS,
+          max_hash_keys: DEFAULT_MAX_HASH_KEYS,
+          max_depth_value: MAX_DEPTH_VALUE,
+          truncation_key: TRUNCATION_METADATA_KEY.to_sym,
+          track_paths: nil,
+          &block
+        )
+          super(
+            max_array_items: max_array_items,
+            max_depth: max_depth,
+            max_depth_value: max_depth_value,
+            max_hash_keys: max_hash_keys,
+            max_string_bytes: max_string_bytes,
+            truncation_key: truncation_key
+          )
+          @transform = block
+          @prepare_values = !@transform.nil?
+          @track_paths = @prepare_values && !track_paths.equal?(false)
+        end
+
+        def call(value)
+          @root = value
+          walk(value)
+        ensure
+          @root = nil
+        end
+
+        private
+
+        def prepare_value(value, depth, key, path)
+          transformed = @transform.call(value, key: key, path: path, original: @root, depth: depth)
+          transformed.equal?(CONTINUE) ? value : transformed
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/serialization/bounded_traversal.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    module Serialization
+      class BoundedTraversal
+        include ValueTraversal
+
+        MAX_DEPTH_VALUE = "[MaxDepth]"
+        TRUNCATED_SUFFIX = "...[Truncated]"
+        TRUNCATION_METADATA_KEY = "_julewire_truncation"
+        DEFAULT_MAX_DEPTH = 8
+        DEFAULT_MAX_STRING_BYTES = 16_384
+        DEFAULT_MAX_ARRAY_ITEMS = 1_000
+        DEFAULT_MAX_HASH_KEYS = 1_000
+
+        class << self
+          def truncation_metadata(
+            fields,
+            max_array_items: DEFAULT_MAX_ARRAY_ITEMS,
+            max_depth: DEFAULT_MAX_DEPTH,
+            max_hash_keys: DEFAULT_MAX_HASH_KEYS,
+            max_string_bytes: DEFAULT_MAX_STRING_BYTES
+          )
+            {
+              "truncated" => true,
+              "truncated_fields" => Array(fields).uniq,
+              "limits" => {
+                "max_array_items" => max_array_items,
+                "max_depth" => max_depth,
+                "max_hash_keys" => max_hash_keys,
+                "max_string_bytes" => max_string_bytes
+              }
+            }
+          end
+        end
+
+        def initialize(max_array_items:, max_depth:, max_depth_value:, max_hash_keys:, max_string_bytes:,
+                       truncation_key:)
+          @max_array_items = Validation.validate_integer_limit!(max_array_items, name: :max_array_items)
+          @max_depth = Validation.validate_integer_limit!(max_depth, name: :max_depth, positive: true)
+          @max_depth_value = max_depth_value
+          @max_hash_keys = Validation.validate_integer_limit!(max_hash_keys, name: :max_hash_keys)
+          @max_string_bytes = Validation.validate_integer_limit!(max_string_bytes, name: :max_string_bytes)
+          @truncation_key = truncation_key
+          @last_truncated = false
+          @compact_empty = false
+          @prepare_values = false
+          @track_paths = false
+        end
+
+        private
+
+        def walk(value)
+          @last_truncated = false
+          traverse(value) { |root, depth| walk_value(root, depth, nil, nil) }
+        ensure
+          @last_truncated = false
+        end
+
+        def walk_value(value, depth, key, path)
+          @last_truncated = false
+          value = prepare_value(value, depth, key, path) if @prepare_values
+          return max_depth_value if depth >= @max_depth
+          return walk_container(value, depth, path) if value.is_a?(Array) || hash_like?(value)
+
+          scalar_value(value, depth, key, path)
+        rescue StandardError => e
+          @last_truncated = false
+          error_value(e)
+        end
+
+        def prepare_value(value, _depth, _key, _path) = value
+
+        def hash_like?(value) = value.is_a?(Hash)
+
+        def scalar_value(value, _depth, _key, _path)
+          value.is_a?(String) ? string_value(value) : clear_truncated(value)
+        end
+
+        # Transform-stage errors must bubble so processors can fail closed.
+        def error_value(error)
+          raise error
+        end
+
+        def walk_container(value, depth, path)
+          return circular_value if traversal_seen?(value)
+
+          with_marked_traversal_container(value) do
+            value.is_a?(Array) ? walk_array(value, depth, path) : walk_hash(value, depth, path)
+          end
+        end
+
+        def circular_value
+          @last_truncated = true
+          Core::CIRCULAR_REFERENCE
+        end
+
+        def max_depth_value
+          mark_truncated(copy_string(@max_depth_value))
+        end
+
+        def walk_hash(value, depth, path)
+          return walk_compact_hash(value, depth, path) if @compact_empty
+
+          walk_full_hash(value, depth, path)
+        end
+
+        def walk_full_hash(value, depth, path)
+          fields = nil
+          result = {}
+          track_paths = @track_paths
+          value.each do |raw_key, item|
+            if result.length >= @max_hash_keys
+              fields = append_truncation_field(fields, "hash_keys")
+              break
+            end
+
+            child = walk_value(item, depth + 1, raw_key, track_paths ? path_for(path, raw_key) : nil)
+            child_truncated = consume_truncated
+            key = key_value(raw_key)
+            key_truncated = consume_truncated
+            result[key] = child
+            fields = record_hash_truncation(fields, raw_key, key, key_truncated, child_truncated)
+          end
+          finish_hash(result, fields)
+        end
+
+        def walk_compact_hash(value, depth, path)
+          fields = nil
+          result = {}
+          track_paths = @track_paths
+          value.each do |raw_key, item|
+            next if raw_omitted_value?(item)
+
+            child = walk_value(item, depth + 1, raw_key, track_paths ? path_for(path, raw_key) : nil)
+            child_truncated = consume_truncated
+            next if omitted_value?(child)
+
+            if result.length >= @max_hash_keys
+              fields = append_truncation_field(fields, "hash_keys")
+              break
+            end
+
+            key = key_value(raw_key)
+            key_truncated = consume_truncated
+            result[key] = child
+            fields = record_hash_truncation(fields, raw_key, key, key_truncated, child_truncated)
+          end
+          finish_hash(result, fields)
+        end
+
+        def walk_array(value, depth, path)
+          return walk_compact_array(value, depth, path) if @compact_empty
+
+          walk_full_array(value, depth, path)
+        end
+
+        def walk_full_array(value, depth, path)
+          fields = nil
+          result = []
+          value.each do |item|
+            if result.length >= @max_array_items
+              fields = append_truncation_field(fields, "array_items")
+              break
+            end
+
+            child = walk_value(item, depth + 1, nil, path)
+            child_truncated = consume_truncated
+            result << child
+            fields = append_truncation_field(fields, "array_items") if child_truncated
+          end
+          finish_array(result, fields)
+        end
+
+        def walk_compact_array(value, depth, path)
+          fields = nil
+          result = []
+          value.each do |item|
+            next if raw_omitted_value?(item)
+
+            child = walk_value(item, depth + 1, nil, path)
+            child_truncated = consume_truncated
+            next if omitted_value?(child)
+
+            if result.length >= @max_array_items
+              fields = append_truncation_field(fields, "array_items")
+              break
+            end
+
+            result << child
+            fields = append_truncation_field(fields, "array_items") if child_truncated
+          end
+          finish_array(result, fields)
+        end
+
+        def omitted_value?(_value) = false
+
+        def raw_omitted_value?(_value) = false
+
+        def key_value(key)
+          clear_truncated(key.is_a?(String) ? copy_string(key) : key)
+        end
+
+        def record_hash_truncation(fields, raw_key, _key, _key_truncated, child_truncated)
+          return fields unless child_truncated
+
+          append_truncation_field(fields, raw_key.to_s)
+        end
+
+        def finish_hash(result, fields)
+          return clear_truncated(result) unless fields
+
+          result[@truncation_key] = truncation_metadata(fields) if @truncation_key
+          mark_truncated(result)
+        end
+
+        def finish_array(result, fields)
+          return clear_truncated(result) unless fields
+
+          result << { @truncation_key => truncation_metadata(fields) } if @truncation_key
+          mark_truncated(result)
+        end
+
+        def truncation_metadata(fields)
+          self.class.truncation_metadata(
+            fields,
+            max_array_items: @max_array_items,
+            max_depth: @max_depth,
+            max_hash_keys: @max_hash_keys,
+            max_string_bytes: @max_string_bytes
+          )
+        end
+
+        def string_value(value)
+          return clear_truncated(copy_string(value)) if value.bytesize <= @max_string_bytes
+
+          mark_truncated("#{value.byteslice(0, @max_string_bytes).scrub("?")}#{TRUNCATED_SUFFIX}")
+        end
+
+        def mark_truncated(value)
+          @last_truncated = true
+          value
+        end
+
+        def clear_truncated(value)
+          @last_truncated = false
+          value
+        end
+
+        def consume_truncated
+          truncated = @last_truncated
+          @last_truncated = false
+          truncated
+        end
+
+        def copy_string(value)
+          value.is_a?(String) && !value.frozen? ? value.dup : value
+        end
+
+        def append_truncation_field(fields, field)
+          (fields ||= []) << field
+          fields
+        end
+
+        def path_for(parent_path, key)
+          parent_path ? "#{parent_path}.#{key}" : key.to_s
+        end
+      end
+
+      private_constant :BoundedTraversal
+    end
+  end
+end