rlm-rb 0.1.0 → 0.2.0

data/lib/rlm/runtime.rb

@@ -0,0 +1,352 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "code_extractor"
+require_relative "context"
+require_relative "errors"
+require_relative "file"
+require_relative "limits"
+require_relative "prompt_builder"
+require_relative "result"
+require_relative "runtime/bridge"
+require_relative "runtime/signature_registry"
+require_relative "signature"
+require_relative "trace"
+
+module RLM
+  # rubocop:disable Metrics/ClassLength
+  class Runtime
+    def initialize(
+      signature:,
+      input:,
+      lm:,
+      sandbox:,
+      limits:,
+      sub_lm: nil,
+      context: nil,
+      tools: [],
+      skills: [],
+      validators: [],
+      signatures: [],
+      depth: 0,
+      trace_store: nil
+    )
+      @signature = signature
+      @input = input || {}
+      @lm = lm
+      @sub_lm = sub_lm || lm
+      @sandbox = sandbox
+      @limits = limits || Limits.new
+      @context = context || build_context(@input)
+      @tools = Array(tools)
+      @skills = Array(skills)
+      @validators = Array(validators)
+      @signatures = SignatureRegistry.build(signature, signatures)
+      @depth = depth
+      @trace_store = trace_store
+      @trace = Trace.new
+      @iterations = 0
+      @llm_calls = 0
+      @sub_lm_calls = 0
+      @tool_calls = 0
+      @last_submitted_output = nil
+    end
+
+    def call
+      raise ProviderError, "root LM is required" if lm.nil?
+
+      start_run
+      bridge = prepare_sandbox
+      run_loop(bridge)
+    rescue BudgetExceededError => e
+      budget_exceeded_result(e)
+    rescue ToolError => e
+      finish(:tool_error, error: e)
+    rescue ProviderError => e
+      finish(:provider_error, error: e)
+    rescue SandboxError => e
+      finish(:sandbox_error, error: e)
+    rescue ValidationError => e
+      validation_failure([e.message], e)
+    rescue ParseError, ConfigurationError => e
+      finish(:aborted, error: e)
+    ensure
+      sandbox&.cleanup
+    end
+
+    def predict_subcall(signature, input, depth:)
+      raise BudgetExceededError, "max_recursion_depth exceeded" if depth > limits.max_recursion_depth
+      raise ProviderError, "sub LM is required" if sub_lm.nil?
+
+      parsed = call_sub_lm(signature, input, depth)
+      raise ValidationError, "sub LM must return <rlm-final> in v0.2 mock runtime" unless parsed.final?
+
+      output = Signature.coerce_output(signature, parsed.content)
+      validate_output!(signature, output)
+      output
+    end
+
+    def record_tool_attempt!
+      trace.record(:budget_checked, budget: :tool_calls, current: @tool_calls, limit: limits.max_tool_calls)
+      raise BudgetExceededError, "max_tool_calls exceeded" if @tool_calls >= limits.max_tool_calls
+
+      @tool_calls += 1
+    end
+
+    def record_submitted_output(output)
+      @last_submitted_output = output
+    end
+
+    private
+
+    attr_reader :signature, :input, :lm, :sub_lm, :context, :tools, :skills,
+                :sandbox, :limits, :validators, :signatures, :depth, :trace,
+                :iterations, :llm_calls, :sub_lm_calls, :trace_store
+
+    def start_run
+      trace.record(:run_started, signature: Signature.name_for(signature), input: input)
+      validate_root_input!
+    end
+
+    def prepare_sandbox
+      bridge = Bridge.new(
+        runtime: self,
+        context: context,
+        trace: trace,
+        tools: tools,
+        signatures: signatures,
+        depth: depth
+      )
+      sandbox.prepare(context: context, tools: tools, skills: skills, runtime_bridge: bridge)
+      bridge
+    end
+
+    def run_loop(bridge)
+      loop do
+        ensure_time_budget!
+        parsed = call_root_lm
+        return complete(parsed.content) if parsed.final?
+
+        execute_code(parsed.content)
+        return complete(bridge.submitted_output) unless bridge.submitted_output.nil?
+      end
+    end
+
+    def call_root_lm
+      ensure_llm_budget!
+      prompt = PromptBuilder.build(signature, input: input, context: context, limits: limits)
+      trace.record(:root_prompt_created, bytes: prompt.bytesize)
+      response = call_lm(lm, :root_lm_called, signature, prompt, depth)
+      CodeExtractor.extract(response)
+    end
+
+    def call_sub_lm(checked_signature, payload, sub_depth)
+      ensure_llm_budget!
+      ensure_sub_lm_budget!
+      prompt = PromptBuilder.build(checked_signature, input: payload, context: context, limits: limits)
+      response = call_lm(sub_lm, :sub_lm_called, checked_signature, prompt, sub_depth)
+      @sub_lm_calls += 1
+      CodeExtractor.extract(response)
+    end
+
+    def call_lm(candidate, event_type, checked_signature, prompt, call_depth)
+      before_cost = candidate.cost_cents if candidate.respond_to?(:cost_cents)
+      response = candidate.call(prompt: prompt, signature: Signature.name_for(checked_signature), depth: call_depth)
+      @llm_calls += 1
+      payload = {
+        signature: Signature.name_for(checked_signature),
+        cost_cents: cost_delta(candidate, before_cost)
+      }
+      payload[:usage] = candidate.last_usage if candidate.respond_to?(:last_usage) && candidate.last_usage
+      trace.record(event_type, payload)
+      ensure_cost_budget!
+      response
+    end
+
+    def execute_code(code)
+      ensure_time_budget!
+      trace.record(:budget_checked, budget: :iterations, current: iterations, limit: limits.max_iterations)
+      raise BudgetExceededError, "max_iterations exceeded" if iterations >= limits.max_iterations
+
+      @iterations += 1
+      trace.record(:code_generated, code: code)
+      result = sandbox.exec(code)
+      ensure_stdout_budget!(result)
+      trace.record(:code_executed, result: result.to_h)
+      handle_sandbox_result!(result)
+    end
+
+    def handle_sandbox_result!(result)
+      return if result.ok?
+
+      case result.error
+      when BudgetExceededError, ParseError, ToolError
+        raise result.error
+      else
+        raise SandboxError, result.error&.message || result.stderr || "sandbox execution failed"
+      end
+    end
+
+    def complete(output)
+      coerced_output = Signature.coerce_output(signature, output)
+      ensure_output_budget!(coerced_output)
+      errors = validate_output(signature, coerced_output)
+      return validation_failure(errors) unless errors.empty?
+
+      trace.record(:run_completed, status: :completed)
+      finish(:completed, output: coerced_output)
+    end
+
+    def validate_root_input!
+      trace.record(:validation_attempted, signature: Signature.name_for(signature), direction: :input)
+      errors = Signature.validate_input(signature, input)
+      return if errors.empty?
+
+      trace.record(:validation_failed, signature: Signature.name_for(signature), direction: :input, errors: errors)
+      raise ValidationError, errors.join(", ")
+    end
+
+    def validate_output!(checked_signature, output)
+      errors = validate_output(checked_signature, output)
+      raise ValidationError, errors.join(", ") unless errors.empty?
+    end
+
+    def validate_output(checked_signature, output)
+      trace.record(:validation_attempted, signature: Signature.name_for(checked_signature), direction: :output)
+      all_errors = Signature.validate_output(checked_signature, output) + custom_validation_errors(output)
+      record_validation_failure(checked_signature, all_errors) unless all_errors.empty?
+      all_errors
+    end
+
+    def custom_validation_errors(output)
+      validators.flat_map { |validator| Array(validator.call(output)) }
+    end
+
+    def record_validation_failure(checked_signature, errors)
+      trace.record(
+        :validation_failed,
+        signature: Signature.name_for(checked_signature),
+        direction: :output,
+        errors: errors
+      )
+    end
+
+    def validation_failure(errors, error = nil)
+      finish(:failed_validation, validation_errors: errors, error: error)
+    end
+
+    def finish(status, output: nil, error: nil, validation_errors: [])
+      record_run_failed(status, error:, validation_errors:) unless status == :completed
+
+      result = Result.new(
+        trace: trace,
+        status: status,
+        output: output,
+        error: error,
+        cost_cents: runtime_cost_cents,
+        duration_ms: trace.duration_ms,
+        llm_calls: llm_calls,
+        iterations: iterations,
+        validation_errors: validation_errors
+      )
+      persist_trace(result)
+      result
+    end
+
+    def ensure_llm_budget!
+      trace.record(:budget_checked, budget: :llm_calls, current: llm_calls, limit: limits.max_llm_calls)
+      raise BudgetExceededError, "max_llm_calls exceeded" if llm_calls >= limits.max_llm_calls
+    end
+
+    def ensure_sub_lm_budget!
+      trace.record(:budget_checked, budget: :sub_lm_calls, current: sub_lm_calls, limit: limits.max_sub_lm_calls)
+      raise BudgetExceededError, "max_sub_lm_calls exceeded" if sub_lm_calls >= limits.max_sub_lm_calls
+    end
+
+    def ensure_cost_budget!
+      current_cost = runtime_cost_cents
+      trace.record(:budget_checked, budget: :cost_cents, current: current_cost, limit: limits.max_cost_cents)
+      raise BudgetExceededError, "max_cost_cents exceeded" if current_cost >= limits.max_cost_cents
+    end
+
+    def ensure_time_budget!
+      current_ms = trace.duration_ms
+      limit_ms = limits.max_runtime_seconds * 1000
+      trace.record(:budget_checked, budget: :runtime_seconds, current: current_ms, limit: limit_ms)
+      raise BudgetExceededError, "max_runtime_seconds exceeded" if current_ms >= limit_ms
+    end
+
+    def ensure_output_budget!(output)
+      current_bytes = JSON.generate(output).bytesize
+      trace.record(:budget_checked, budget: :output_bytes, current: current_bytes, limit: limits.max_output_bytes)
+      raise BudgetExceededError, "max_output_bytes exceeded" if current_bytes > limits.max_output_bytes
+    end
+
+    def ensure_stdout_budget!(result)
+      current_bytes = result.stdout.to_s.bytesize
+      trace.record(:budget_checked, budget: :stdout_bytes, current: current_bytes, limit: limits.max_stdout_bytes)
+      raise BudgetExceededError, "max_stdout_bytes exceeded" if current_bytes > limits.max_stdout_bytes
+    end
+
+    def budget_exceeded_result(error)
+      case limits.on_budget_exceeded
+      when :needs_review
+        finish(:needs_review, output: valid_last_submitted_output, error: error)
+      when :return_partial
+        output = valid_last_submitted_output
+        return finish(:needs_review, output: output, error: error) unless output.nil?
+
+        finish(:budget_exceeded, error: error)
+      else
+        finish(:budget_exceeded, error: error)
+      end
+    end
+
+    def valid_last_submitted_output
+      return if @last_submitted_output.nil?
+      return if validate_output(signature, @last_submitted_output).any?
+
+      ensure_output_budget!(@last_submitted_output)
+      @last_submitted_output
+    rescue BudgetExceededError
+      nil
+    end
+
+    def persist_trace(result)
+      return unless trace_store.respond_to?(:call)
+
+      trace_store.call(result)
+    rescue StandardError
+      nil
+    end
+
+    def record_run_failed(status, error:, validation_errors: [])
+      payload = { status: status }
+      payload[:error] = trace_error_payload(error) if error
+      payload[:errors] = validation_errors if validation_errors.any?
+      trace.record(:run_failed, payload)
+    end
+
+    def trace_error_payload(error)
+      { class: error.class.name, message: error.message }
+    end
+
+    def runtime_cost_cents
+      [lm, sub_lm].compact.uniq.sum do |candidate|
+        candidate.respond_to?(:cost_cents) ? candidate.cost_cents : 0
+      end
+    end
+
+    def cost_delta(candidate, before_cost)
+      return 0 unless candidate.respond_to?(:cost_cents)
+
+      candidate.cost_cents - before_cost.to_i
+    end
+
+    def build_context(payload)
+      Context.new(inputs: payload, files: payload.values.grep(RLM::File))
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/rlm/sandbox/unsafe_in_process.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+# WARNING: This sandbox is intentionally unsafe.
+#
+# It executes model-produced Ruby code with instance_eval inside the current Ruby process.
+# It provides no process isolation, memory isolation, filesystem isolation, network isolation,
+# timeout enforcement, or protection from malicious code. Use it only in development/tests to
+# prove the runtime spine. Production backends must use isolated subprocess, container, or remote
+# runners instead. Stream capture mutates process-global $stdout/$stderr, so this class
+# serializes execution with a mutex; it is still not a concurrency-safe production sandbox.
+
+require "stringio"
+
+module RLM
+  module Sandbox
+    class UnsafeInProcess < Base
+      STREAM_CAPTURE_MUTEX = Mutex.new
+
+      attr_reader :context, :tools, :skills, :runtime_bridge
+
+      def initialize
+        super
+        @prepared = false
+      end
+
+      def prepared?
+        @prepared
+      end
+
+      def prepare(context:, tools:, skills:, runtime_bridge:)
+        @context = context
+        @tools = tools
+        @skills = skills
+        @runtime_bridge = runtime_bridge
+        @prepared = true
+        ExecutionResult.new(status: :ok)
+      end
+
+      def exec(code)
+        raise SandboxError, "Sandbox not prepared" unless prepared?
+
+        execute(code)
+      end
+
+      def execute(code)
+        stdout, stderr = capture_streams do
+          Scope.new(runtime_bridge).instance_eval(code, "(rlm unsafe in-process sandbox)")
+        end
+
+        ExecutionResult.new(status: :ok, stdout: stdout, stderr: stderr)
+      rescue StandardError => e
+        ExecutionResult.new(status: :error, stderr: e.message, error: e, exit_code: 1)
+      end
+
+      def cleanup
+        @context = nil
+        @tools = nil
+        @skills = nil
+        @runtime_bridge = nil
+        @prepared = false
+      end
+
+      private
+
+      def capture_streams
+        STREAM_CAPTURE_MUTEX.synchronize do
+          old_stdout = $stdout
+          old_stderr = $stderr
+          captured_stdout = StringIO.new
+          captured_stderr = StringIO.new
+          $stdout = captured_stdout
+          $stderr = captured_stderr
+          yield
+          [captured_stdout.string, captured_stderr.string]
+        ensure
+          $stdout = old_stdout
+          $stderr = old_stderr
+        end
+      end
+
+      class Scope
+        def initialize(runtime_bridge)
+          @runtime_bridge = runtime_bridge
+        end
+
+        def predict(signature_name, input_hash)
+          runtime_bridge.predict(signature_name, input_hash)
+        end
+
+        def tool(tool_name, input_hash)
+          runtime_bridge.tool(tool_name, input_hash)
+        end
+
+        def submit(output_hash)
+          runtime_bridge.submit(output_hash)
+        end
+
+        def read_file(handle)
+          runtime_bridge.read_file(handle)
+        end
+
+        def list_files
+          runtime_bridge.list_files
+        end
+
+        def log(message)
+          runtime_bridge.log(message)
+        end
+
+        private
+
+        attr_reader :runtime_bridge
+      end
+    end
+  end
+end

data/lib/rlm/signature/dspy.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module RLM
+  module Signature
+    class Dspy
+      TYPE_MAP = {
+        "array" => :array,
+        "boolean" => :boolean,
+        "integer" => :integer,
+        "number" => :number,
+        "object" => :object,
+        "string" => :string
+      }.freeze
+
+      def initialize(signature)
+        @signature = signature
+      end
+
+      def name
+        return signature.name if signature.respond_to?(:name) && !signature.name.to_s.empty?
+
+        signature.to_s
+      end
+
+      def description
+        return signature.description if signature.respond_to?(:description)
+
+        name
+      end
+
+      def input_fields
+        fields_for(input_schema)
+      end
+
+      def output_fields
+        fields_for(output_schema)
+      end
+
+      def validate_input(input)
+        validate_payload(input, input_schema)
+      end
+
+      def validate_output(output)
+        validate_payload(output, output_schema)
+      end
+
+      def coerce_output(output)
+        return output unless output.is_a?(Hash)
+
+        schema_keys = output_fields.keys
+        output.each_with_object({}) do |(key, value), coerced|
+          coerced_key = schema_keys.find { |schema_key| schema_key.to_s == key.to_s } || key
+          coerced[coerced_key] = value
+        end
+      end
+
+      private
+
+      attr_reader :signature
+
+      def input_schema
+        schema_from(:input_json_schema, :input_schema)
+      end
+
+      def output_schema
+        schema_from(:output_json_schema, :output_schema)
+      end
+
+      def schema_from(*method_names)
+        method_names.each do |method_name|
+          next unless signature.respond_to?(method_name)
+
+          schema = signature.public_send(method_name)
+          return normalize_hash(schema) if schema.is_a?(Hash)
+        end
+
+        raise ConfigurationError, "dspy signature #{name} does not expose JSON schema metadata"
+      end
+
+      def fields_for(schema)
+        properties_for(schema).each_with_object({}) do |(field_name, metadata), fields|
+          fields[field_name.to_sym] = field_type(metadata)
+        end
+      end
+
+      def validate_payload(payload, schema)
+        return ["payload must be a Hash"] unless payload.is_a?(Hash)
+
+        normalized = normalize_hash(payload)
+        required_errors(schema, normalized) + type_errors(schema, normalized)
+      end
+
+      def required_errors(schema, payload)
+        required_fields(schema).filter_map do |field_name|
+          "#{field_name} is required" unless payload.key?(field_name.to_s) || payload.key?(field_name.to_sym)
+        end
+      end
+
+      def type_errors(schema, payload)
+        properties_for(schema).filter_map do |field_name, metadata|
+          value = fetch_payload_value(payload, field_name)
+          next if value.nil? || value_matches_type?(value, field_type(metadata))
+
+          "#{field_name} must be #{field_type(metadata)}"
+        end
+      end
+
+      def fetch_payload_value(payload, field_name)
+        payload.fetch(field_name.to_s) { payload[field_name.to_sym] }
+      end
+
+      def required_fields(schema)
+        Array(schema["required"] || schema[:required]).map(&:to_s)
+      end
+
+      def properties_for(schema)
+        normalize_hash(schema["properties"] || schema[:properties] || {})
+      end
+
+      def field_type(metadata)
+        normalized = normalize_hash(metadata || {})
+        type = normalized["type"] || normalized[:type]
+        TYPE_MAP.fetch(type.to_s, type.to_s.empty? ? :object : type.to_s.to_sym)
+      end
+
+      def value_matches_type?(value, type)
+        case type
+        when :array
+          value.is_a?(Array)
+        when :boolean
+          boolean?(value)
+        when :integer
+          value.is_a?(Integer)
+        when :number
+          value.is_a?(Numeric)
+        when :object
+          value.is_a?(Hash)
+        when :string
+          value.is_a?(String)
+        else true
+        end
+      end
+
+      def boolean?(value)
+        [true, false].include?(value)
+      end
+
+      def normalize_hash(hash)
+        hash.each_with_object({}) do |(key, value), normalized|
+          normalized[key] = value.is_a?(Hash) ? normalize_hash(value) : value
+        end
+      end
+    end
+  end
+end

data/lib/rlm/signature.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+
+module RLM
+  module Signature
+    REQUIRED_METHODS = %i[
+      description
+      input_fields
+      output_fields
+      validate_input
+      validate_output
+    ].freeze
+
+    module_function
+
+    def validate_interface!(signature)
+      missing = REQUIRED_METHODS.reject { |method_name| signature.respond_to?(method_name) }
+      raise ConfigurationError, "signature is missing required methods: #{missing.join(", ")}" unless missing.empty?
+
+      validate_fields!(signature, :input_fields)
+      validate_fields!(signature, :output_fields)
+      signature
+    end
+
+    def validate_input(signature, input)
+      validate_payload(signature, input, :validate_input)
+    end
+
+    def validate_output(signature, output)
+      validate_payload(signature, output, :validate_output)
+    end
+
+    def coerce_output(signature, output)
+      return signature.coerce_output(output) if signature.respond_to?(:coerce_output)
+
+      output
+    end
+
+    def assert_valid_input!(signature, input)
+      errors = validate_input(signature, input)
+      raise ValidationError, errors.join(", ") unless errors.empty?
+
+      nil
+    end
+
+    def assert_valid_output!(signature, output)
+      errors = validate_output(signature, output)
+      raise ValidationError, errors.join(", ") unless errors.empty?
+
+      nil
+    end
+
+    def name_for(signature)
+      name = signature.name if signature.respond_to?(:name)
+      return name unless name.to_s.empty?
+
+      signature.to_s
+    end
+
+    def validate_fields!(signature, method_name)
+      fields = signature.public_send(method_name)
+      return if fields.is_a?(Hash)
+
+      raise ConfigurationError, "signature .#{method_name} must return a Hash"
+    end
+
+    def validate_payload(signature, payload, method_name)
+      validate_interface!(signature)
+      errors = signature.public_send(method_name, payload)
+      return errors if errors.is_a?(Array)
+
+      raise ConfigurationError, "signature .#{method_name} must return an Array"
+    end
+  end
+end

data/lib/rlm/trace.rb

@@ -15,6 +15,8 @@ module RLM
       file_read
       tool_called
       sub_lm_called
+      output_submitted
+      runtime_logged
       validation_attempted
       validation_failed
       budget_checked