nitro_intelligence 3.1.0 → 4.1.0

data/lib/nitro_intelligence/models/model_catalog.rb

@@ -1,9 +1,11 @@
 require "nitro_intelligence/models/model_factory"
 
 module NitroIntelligence
+  # @api private
   class ModelCatalog
     attr_reader :models,
                 :default_audio_transcription_model,
+                :default_decision_model,
                 :default_image_model,
                 :default_text_model,
                 :default_text_to_speech_model
@@ -12,6 +14,7 @@ module NitroIntelligence
       model_config = model_config.symbolize_keys
       @models = (model_config[:models] || []).map { |model_metadata| ModelFactory.build(model_metadata) }
       @default_audio_transcription_model = lookup_by_name(model_config[:default_audio_transcription_model])
+      @default_decision_model = lookup_by_name(model_config[:default_decision_model])
       @default_image_model = lookup_by_name(model_config[:default_image_model])
       @default_text_model = lookup_by_name(model_config[:default_text_model])
       @default_text_to_speech_model = lookup_by_name(model_config[:default_text_to_speech_model])

data/lib/nitro_intelligence/models/model_factory.rb

@@ -1,10 +1,12 @@
 require "nitro_intelligence/models/model"
 
 module NitroIntelligence
+  # @api private
   class ModelFactory
     TYPES = {
       "text" => TextModel,
       "audio_transcription" => TextModel,
+      "decision" => DecisionModel,
       "image" => ImageModel,
       "text_to_speech" => TextToSpeechModel,
     }.freeze

data/lib/nitro_intelligence/null_cache.rb

@@ -1,6 +1,8 @@
 module NitroIntelligence
   # Default no-operation cache implementation used
   # as a sensible default for initializing NitroIntelligence
+  #
+  # @api private
   class NullCache
     def read(_key) = nil
     def write(_key, _value, **_options) = true

data/lib/nitro_intelligence/observability/project.rb

@@ -2,18 +2,36 @@ require "base64"
 require "nitro_intelligence/observability/prompt_store"
 
 module NitroIntelligence
+  # Cerebro, the Langfuse deployment observed requests are recorded in.
   module Observability
+    # A Cerebro project `observability_projects` configures. Use it to reach Cerebro's API, or to
+    # link to the project's traces, where this gem does not.
     class Project
+      # Raised by {ProjectClientRegistry#fetch} when `observability_projects` does not configure the
+      # project.
       class NotFoundError < StandardError; end
 
-      attr_reader :slug, :id, :public_key, :secret_key, :auth_token, :prompt_store
+      # @api private
+      # Recorded in place of anything a masked project would otherwise send to Cerebro.
+      MASKED_CONTENT = "[masked]".freeze
 
-      def initialize(slug:, id:, public_key:, secret_key:, **_kwargs)
+      # @return [String] the project's id in Cerebro, as in its URLs:
+      #   `"#{config.observability_base_url}/project/#{id}/traces/#{trace_id}"`
+      attr_reader :id
+      # @return [String] the Base64 `public_key:secret_key` credential for Cerebro's public API,
+      #   without the scheme: send it as `Authorization: Basic #{auth_token}`
+      attr_reader :auth_token
+      # @api private
+      attr_reader :slug, :public_key, :secret_key, :prompt_store
+
+      # @api private
+      def initialize(slug:, id:, public_key:, secret_key:, mask_content: false, **_kwargs)
         @slug = slug
         @id = id
         @public_key = public_key
         @secret_key = secret_key
         @auth_token = Base64.strict_encode64("#{public_key}:#{secret_key}")
+        @mask_content = mask_content == true
         @prompt_store = PromptStore.new(
           observability_project_slug: slug,
           observability_public_key: public_key,
@@ -21,6 +39,27 @@ module NitroIntelligence
         )
       end
 
+      # @api private
+      def mask_content?
+        @mask_content
+      end
+
+      # @api private
+      #
+      # Replaces the whole value rather than fields within it: the shape of a response
+      # varies by provider, and redacting named fields is what let reasoning leak
+      # through the gateway's own redaction. Absence stays absence, so an observation
+      # that never had an output does not gain one.
+      def mask(value)
+        return value if !mask_content? || value.nil?
+
+        MASKED_CONTENT
+      end
+
+      # The project `observability_projects` configures under `slug`.
+      #
+      # @param slug [String]
+      # @return [Project, nil] nil when no project is configured under `slug`
       def self.find_by_slug(slug:)
         project_config = NitroIntelligence.config.observability_projects.find { |project| project["slug"] == slug }
 

data/lib/nitro_intelligence/observability/project_client.rb

@@ -1,15 +1,24 @@
 module NitroIntelligence
   module Observability
+    # A project's connection to Cerebro, as {ProjectClientRegistry#fetch} gives it.
     class ProjectClient
+      # Not raised by the gem, which raises {Project::NotFoundError} for a project
+      # `observability_projects` does not configure. Public only because applications raise it
+      # themselves.
       class NotFoundError < StandardError; end
 
-      attr_reader :project, :observability_client
+      # @return [Project]
+      attr_reader :project
+      # @api private
+      attr_reader :observability_client
 
+      # @api private
       def initialize(project:, observability_client:)
         @project = project
         @observability_client = observability_client
       end
 
+      # @api private
       def shutdown
         @observability_client.shutdown
       end

data/lib/nitro_intelligence/observability/project_client_registry.rb

@@ -4,16 +4,34 @@ require "nitro_intelligence/observability/project_client"
 
 module NitroIntelligence
   module Observability
+    # Each Cerebro project's client, built once per process, as {NitroIntelligence.project_client_registry}
+    # holds them. Fetching each project at boot builds its client before the first request.
+    #
+    # @example In an initializer
+    #   NitroIntelligence.config.observability_projects.each do |project|
+    #     NitroIntelligence.project_client_registry.fetch(project["slug"])
+    #   end
+    #   at_exit { NitroIntelligence.project_client_registry.shutdown_all }
     class ProjectClientRegistry
+      # @api private
       def initialize(base_url:)
         @base_url = base_url
         @project_clients = {}
       end
 
+      # The client for the project configured under `slug`, built on first use.
+      #
+      # @param slug [String]
+      # @return [ProjectClient]
+      # @raise [Project::NotFoundError] when `observability_projects` does not configure the project
       def fetch(slug)
         @project_clients[slug] ||= build_project_client(slug)
       end
 
+      # Sends what each client still holds to Cerebro, and stops it. Call it as the process exits, or
+      # the last traces and scores are lost.
+      #
+      # @return [void]
       def shutdown_all
         @project_clients.values.each(&:shutdown)
       end

data/lib/nitro_intelligence/observability/prompt.rb

@@ -1,5 +1,6 @@
 module NitroIntelligence
   module Observability
+    # @api private
     class Prompt
       attr_reader :name, :type, :prompt, :version, :config, :labels, :tags
 
@@ -21,9 +22,9 @@ module NitroIntelligence
       def compile(**replacements)
         return replace_variables(@prompt, **replacements) if @type == "text"
 
+        # New messages, so the prompt keeps its variables for the next request that compiles it.
         @prompt.map do |message|
-          message[:content] = replace_variables(message[:content], **replacements)
-          message
+          message.merge(content: replace_variables(message[:content], **replacements))
         end
       end
 
@@ -31,7 +32,7 @@ module NitroIntelligence
       # into the correct position based on prompt "type" ("text" or "chat")
       def interpolate(messages:, variables:)
         if @type == "text"
-          messages.prepend({ role: "system", content: compile(**variables) })
+          [{ role: "system", content: compile(**variables) }] + messages
         elsif @type == "chat"
           compile(**variables) + messages
         end

data/lib/nitro_intelligence/observability/prompt_resolver.rb

@@ -4,6 +4,8 @@ module NitroIntelligence
   module Observability
     # Resolves a managed prompt from the lookups a request asks for, most specific first. A lookup that is
     # missing -- or that fails -- yields to the next; the last one's failure propagates.
+    #
+    # @api private
     class PromptResolver
       # One prompt lookup: the name asked for, at a label or a version.
       Lookup = Data.define(:name, :label, :version)

data/lib/nitro_intelligence/observability/prompt_store.rb

@@ -7,10 +7,19 @@ require "nitro_intelligence/observability/prompt"
 
 module NitroIntelligence
   module Observability
+    # @api private
     class PromptStore
       OBSERVABILITY_PROMPTS_CACHE_KEY_PREFIX = "nitro_intelligence_observability_prompts_".freeze
 
+      # Raised by an observed request when Cerebro fails to return the prompt it names. The message
+      # is the response body.
+      #
+      # @api public
       class ObservabilityPromptError < StandardError; end
+      # Raised by an observed request when the prompt it names, and its fallback if it names one,
+      # does not exist.
+      #
+      # @api public
       class ObservabilityPromptNotFoundError < StandardError; end
 
       def initialize(observability_project_slug:, observability_public_key:, observability_secret_key:)

data/lib/nitro_intelligence/observability/upload_handler.rb

@@ -7,6 +7,7 @@ require "uri"
 
 module NitroIntelligence
   module Observability
+    # @api private
     class UploadHandler
       attr_reader :uploaded_media
 
@@ -58,7 +59,7 @@ module NitroIntelligence
         payload
       end
 
-      def upload(trace_id, upload_queue: Queue.new)
+      def upload(trace_id, observation_id:, upload_queue: Queue.new)
         until upload_queue.empty?
           media = upload_queue.pop
 
@@ -68,6 +69,7 @@ module NitroIntelligence
           # returns {"mediaId" -> "", "uploadUrl" => ""}
           upload_url_response = get_upload_url({
                                                  traceId: trace_id,
+                                                 observationId: observation_id,
                                                  contentType: media.mime_type,
                                                  contentLength: content_length,
                                                  sha256Hash: content_sha256,

data/lib/nitro_intelligence/pending_review.rb

@@ -0,0 +1,257 @@
+require "active_support/core_ext/hash/indifferent_access"
+require "nitro_intelligence/review_decision"
+require "nitro_intelligence/tool_call_review_interrupt"
+require "nitro_intelligence/tool_call_review_validator"
+
+module NitroIntelligence
+  # What a paused run is waiting on: the interrupt the thread is holding, and the decisions that
+  # answer it.
+  #
+  # Assistants pauses for two kinds of review, never mixed in one interrupt:
+  #
+  # - tool calls, held by LangChain's `HumanInTheLoopMiddleware`, one action per call;
+  # - the run's final reply, a single action named `__reply__` whose `args` carry the draft's
+  #   `content`.
+  #
+  # A turn with tool calls only ever pauses on its tool calls, and the reply is the run's final
+  # message, so `#tool_calls` and `#reply` are never both present.
+  #
+  # Built from a thread state (`GET /threads/{id}/state`, which reports `interrupts`) or from the
+  # response of a paused `/runs/wait` (which reports `__interrupt__`); both carry the messages the
+  # tool calls are matched back onto.
+  class PendingReview
+    # @api private
+    REPLY_ACTION = "__reply__".freeze
+
+    # @api private
+    # What a reviewer may do with a reply. LangChain's `respond` supplies a tool's result, which a
+    # reply does not have, so it is never offered for one, whatever the interrupt lists.
+    REPLY_DECISIONS = %w[approve edit reject].freeze
+
+    # @api private
+    KIND_NAMES = { tool_calls: "tool calls", reply: "a reply" }.freeze
+
+    # The draft reply a reviewer is asked to approve, edit or reject.
+    #
+    # @!attribute [r] content
+    #   @return [String] the draft
+    # @!attribute [r] allowed_decisions
+    #   @return [Array<String>] what the reviewer may do, from `approve`, `edit` and `reject`
+    # @!attribute [r] description
+    #   @return [String, nil] the guidance the pause gives the reviewer
+    # @!attribute [r] interrupt_id
+    #   @return [String] the pause, as {PendingReview#interrupt_id}
+    Reply = Struct.new(:content, :allowed_decisions, :description, :interrupt_id, keyword_init: true)
+
+    # The pending review in a thread state, or nil when the thread is not paused.
+    #
+    # @api private
+    def self.from_thread_state(thread_state)
+      interrupt = Array(thread_state["interrupts"]).first
+      interrupt && new(interrupt:, messages: Array(thread_state.dig("values", "messages")))
+    end
+
+    # The pending review in a `/runs/wait` response, or nil when the run did not pause.
+    #
+    # @api private
+    def self.from_run(run)
+      interrupt = Array(run["__interrupt__"]).first
+      interrupt && new(interrupt:, messages: Array(run["messages"]))
+    end
+
+    # @api private
+    def initialize(interrupt:, messages:)
+      @interrupt = interrupt.to_h
+      @messages = messages
+    end
+
+    # Identifies this pause. A thread that has moved on -- a new message superseded the review, or
+    # another reviewer decided it first -- reports a different interrupt, or none.
+    #
+    # @return [String, nil] nil for a tool-call pause the service reported without an id, which
+    #   {Assistants#review} then resumes without checking that it is still the same pause
+    def interrupt_id
+      @interrupt["id"] || @interrupt["interrupt_id"]
+    end
+
+    # What the run is paused on.
+    #
+    # @return [Symbol] `:tool_calls` or `:reply`
+    def kind
+      action_name == REPLY_ACTION ? :reply : :tool_calls
+    end
+
+    # @return [Boolean] whether the run is paused on its tool calls
+    def tool_calls?
+      kind == :tool_calls
+    end
+
+    # @return [Boolean] whether the run is paused on its reply
+    def reply?
+      kind == :reply
+    end
+
+    # The tool calls the interrupt is holding, as `Assistants#tool_calls_under_review` reports them.
+    # Empty when the pause is on a reply.
+    #
+    # @return [Array<Hash>] each with `previous_message_id`, `id`, `name`, `args` and
+    #   `allowed_decisions`
+    def tool_calls
+      return [] unless tool_calls?
+
+      tool_call_interrupt.tool_calls
+    end
+
+    # The held draft, or nil when the pause is not on a reply.
+    #
+    # @return [Reply, nil]
+    def reply
+      return nil unless reply?
+
+      @reply ||= Reply.new(
+        content: action_args["content"],
+        allowed_decisions:,
+        description: action_requests.first["description"],
+        interrupt_id:
+      )
+    end
+
+    # The `command.resume` that sends `decisions` to this pause, keyed by its interrupt. LangGraph
+    # applies a keyed resume to that interrupt only: keyed by one that is no longer pending -- the
+    # thread moved on between reading this pause and resuming it -- it decides nothing, and the
+    # thread stays paused on the newer interrupt. An interrupt reported without an id can only be
+    # resumed unkeyed.
+    #
+    # @api private
+    def resume(decisions)
+      interrupt_id.blank? ? { decisions: } : { interrupt_id => { decisions: } }
+    end
+
+    # Whether `run`, the response to a keyed resume of this pause, decided nothing because the
+    # interrupt was gone by the time the resume arrived. `run_id` is the run Assistants created for
+    # the resume, and the block reads the thread's state, only when it is needed.
+    #
+    # Such a resume comes back paused on the newer interrupt, exactly as a resume that was applied
+    # and paused again does -- a turn's tool calls decided, then the run's reply held. What tells
+    # them apart is who wrote the thread's latest checkpoint: Assistants records each run's id in
+    # the metadata of the checkpoints it writes, and a resume that decided nothing writes none, so
+    # the pause it reports still sits on a checkpoint an earlier run wrote.
+    #
+    # Best-effort, and only ever true on that evidence: without the run's id, the checkpoint's, or a
+    # state still on the reported pause (a new message may already have moved it on), or when the
+    # state cannot be read, it is false and the result stands as returned. That a stale decision is
+    # never applied is guaranteed by the keyed resume, not by this.
+    #
+    # @api private
+    def unapplied_resume?(run:, run_id:)
+      paused_on = self.class.from_run(run)&.interrupt_id
+      return false if interrupt_id.blank? || run_id.blank? || paused_on.blank? || paused_on == interrupt_id
+
+      state = yield
+      writer = state.dig("metadata", "run_id")
+      self.class.from_thread_state(state)&.interrupt_id == paused_on && writer.present? && writer != run_id
+    rescue
+      false
+    end
+
+    # The `command.resume` decisions for a review of this pause, validated against what the
+    # interrupt allows before anything is built, raising `Assistants::ThreadResumptionError`.
+    #
+    # For a reply, `reviews` is a single review hash. For tool calls, it is either a hash of reviews
+    # keyed by tool call id, as `Assistants#review_tool_calls` has always taken them, or a list of
+    # reviews that each carry their tool call's `id`.
+    #
+    # @api private
+    def decisions(reviews)
+      return [reply_decision(reviews)] if reply?
+
+      reviews = tool_call_reviews(reviews)
+      ToolCallReviewValidator.new.validate!(tool_calls: reviews, tool_calls_under_review: tool_calls)
+      tool_call_interrupt.decisions(reviews)
+    end
+
+  private
+
+    def tool_call_interrupt
+      @tool_call_interrupt ||= ToolCallReviewInterrupt.new(
+        "interrupts" => [@interrupt],
+        "values" => { "messages" => @messages }
+      )
+    end
+
+    def interrupt_value
+      @interrupt["value"] || {}
+    end
+
+    def action_requests
+      Array(interrupt_value["action_requests"])
+    end
+
+    def action_name
+      action_requests.first&.dig("name")
+    end
+
+    def action_args
+      action_requests.first["args"] || {}
+    end
+
+    def allowed_decisions
+      review_config = Array(interrupt_value["review_configs"]).find do |config|
+        config["action_name"] == action_name
+      end
+
+      Array(review_config&.dig("allowed_decisions")).map(&:to_s) & REPLY_DECISIONS
+    end
+
+    def tool_call_reviews(reviews)
+      return reviews unless reviews.is_a?(Array)
+
+      reviews.each_with_object({}) do |review, by_id|
+        unless review.is_a?(Hash) && review.with_indifferent_access[:id].present?
+          raise_error!("Each tool call review in a list must be a hash with the tool call's `id`")
+        end
+
+        review = review.with_indifferent_access
+        id = review[:id].to_s
+        raise_error!("Tool call `#{id}` is reviewed more than once") if by_id.key?(id)
+
+        by_id[id] = review.except(:id)
+      end
+    end
+
+    # A reply is a single action, so it is reviewed with a single hash.
+    def reply_decision(review)
+      raise_error!("The reply review must be a hash") unless review.is_a?(Hash)
+
+      review = review.with_indifferent_access
+      action = review[:action].to_s
+
+      unless allowed_decisions.include?(action)
+        raise_error!("Invalid review action `#{action}` for the reply; allowed: #{allowed_decisions.join(', ')}")
+      end
+
+      case action
+      when "approve" then ReviewDecision.approve
+      when "edit"
+        ReviewDecision.edit(REPLY_ACTION, { "content" => text!(review, :content, action:, required: true) })
+      when "reject" then ReviewDecision.reject(text!(review, :message, action:, required: false))
+      end
+    end
+
+    # A required field must be non-blank text; an optional one may be absent, but not something other
+    # than a String.
+    def text!(review, field, action:, required:)
+      value = review[field]
+      case ReviewDecision.message_problem(value, required:)
+      when :missing then raise_error!("A `#{action}` review of the reply must include a non-empty `#{field}`")
+      when :not_string then raise_error!("`#{field}` for the reply review must be a string")
+      end
+
+      value
+    end
+
+    def raise_error!(message)
+      raise Assistants::ThreadResumptionError, message
+    end
+  end
+end

data/lib/nitro_intelligence/railtie.rb

@@ -0,0 +1,10 @@
+require "nitro_intelligence/deprecator"
+
+module NitroIntelligence
+  # @api private
+  class Railtie < ::Rails::Railtie
+    initializer "nitro_intelligence.deprecator" do |app|
+      app.deprecators[:nitro_intelligence] = NitroIntelligence.deprecator
+    end
+  end
+end

data/lib/nitro_intelligence/reporter.rb

@@ -3,15 +3,33 @@ require "net/http"
 require "uri"
 
 module NitroIntelligence
+  # Records evaluations of traces in Cerebro.
+  #
+  # @example
+  #   reporter = NitroIntelligence::Reporter.new(observability_project_slug: "my-feature-project")
+  #   reporter.score(name: "precision", value: 0.5, trace_id:)
   class Reporter
+    # Raised by {#create_dataset_item} when Cerebro does not store the item. The message carries the
+    # response's status and body.
     class DatasetItemError < StandardError; end
 
+    # @param observability_project_slug [String] the Cerebro project the traces are recorded in
+    # @raise [Observability::Project::NotFoundError] when `observability_projects` does not configure
+    #   the project
     def initialize(observability_project_slug:)
       @observability_project_slug = observability_project_slug
       @project_client = fetch_project_client
       @host = NitroIntelligence.config.observability_base_url
     end
 
+    # Adds an item to a dataset in the project, for evaluating prompts against later.
+    #
+    # @param attributes [Hash] the item, as Langfuse's `POST /api/public/dataset-items` takes it:
+    #   `datasetName`, `input`, `expectedOutput`, `metadata`, `sourceTraceId` and the rest
+    # @return [Net::HTTPSuccess] the response
+    # @raise [DatasetItemError] when Cerebro does not store the item
+    # @example
+    #   reporter.create_dataset_item({ datasetName: "sms-screening", input: { message: }, expectedOutput: "yes" })
     def create_dataset_item(attributes)
       uri = URI("#{@host}/api/public/dataset-items")
       http = Net::HTTP.new(uri.host, uri.port)
@@ -35,6 +53,17 @@ module NitroIntelligence
       response
     end
 
+    # Scores a trace. Scoring it again under the same `id` replaces the score.
+    #
+    # @param trace_id [String] the trace, such as {Trace.create_id} gives for a seed
+    # @param name [String] what the score measures
+    # @param value [Numeric, String, Boolean] the score: a number, `true` or `false` for `:boolean`,
+    #   or the category's name for `:categorical`
+    # @param id [String] identifies the score, so that recording it again replaces it. Defaults to
+    #   one per trace and name.
+    # @param data_type [Symbol] `:numeric`, `:boolean` or `:categorical`
+    # @return [void]
+    # @raise [ArgumentError] when `value` does not suit `data_type`
     def score(trace_id:, name:, value:, id: "#{trace_id}-#{name}", data_type: :numeric)
       @project_client.observability_client.create_score(
         id:,

data/lib/nitro_intelligence/review_decision.rb

@@ -0,0 +1,36 @@
+module NitroIntelligence
+  # The `command.resume` decisions a review sends, and the check on the `message` a decision
+  # carries, shared by tool call reviews and reply reviews. Each caller keeps its own
+  # error wording, so this reports what is wrong rather than raising.
+  #
+  # @api private
+  module ReviewDecision
+  module_function
+
+    def approve
+      { "type" => "approve" }
+    end
+
+    def edit(name, args)
+      { "type" => "edit", "edited_action" => { "name" => name, "args" => args } }
+    end
+
+    # The middleware falls back to its own wording when a rejection carries no reason.
+    def reject(message)
+      message.nil? ? { "type" => "reject" } : { "type" => "reject", "message" => message }
+    end
+
+    def respond(message)
+      { "type" => "respond", "message" => message }
+    end
+
+    # What is wrong with a decision's `message`, or nil: `:missing` when a required one is absent
+    # or blank, `:not_string` when one is given as anything but a String.
+    def message_problem(message, required:)
+      return (:missing if required) if message.nil?
+      return :not_string unless message.is_a?(String)
+
+      :missing if required && message.blank?
+    end
+  end
+end