xeno 0.0.1

data/app/controllers/xeno/streams_controller.rb

@@ -0,0 +1,122 @@
+module Xeno
+  # The session event stream: SSE by default, NDJSON on request
+  # (`?format=ndjson` or `Accept: application/x-ndjson` — one envelope per
+  # line, friendlier to curl/jq and non-browser consumers). `?start_index=N`
+  # rewinds/resumes — events are durable rows, so reconnecting clients
+  # replay history and then follow live. The stream ends when the session
+  # reaches a terminal status (or the optional configured max duration).
+  #
+  # Known limitation (documented): ActionController::Live holds a thread
+  # per connected client.
+  class StreamsController < ApiController
+    include ActionController::Live
+
+    KEEPALIVE_INTERVAL = 5 # seconds without a write before a ping
+
+    # SSE framing: `id:`/`event:` headers per frame, comment keepalives.
+    class SseWriter
+      def initialize(stream)
+        @stream = stream
+        @sse = ActionController::Live::SSE.new(stream)
+      end
+
+      def content_type = "text/event-stream"
+      def event(record) = @sse.write(record.envelope, event: record.event_type, id: record.index)
+      def keepalive = @stream.write(": keepalive\n\n")
+      def close = @sse.close
+    end
+
+    # NDJSON framing: one envelope per line (type/index live inside the
+    # envelope); a bare newline as keepalive — line parsers skip empties.
+    class NdjsonWriter
+      def initialize(stream)
+        @stream = stream
+      end
+
+      def content_type = "application/x-ndjson"
+      def event(record) = @stream.write("#{JSON.generate(record.envelope)}\n")
+      def keepalive = @stream.write("\n")
+      def close = @stream.close
+    end
+
+    def show
+      session = find_owned_session!
+      writer = build_writer
+      response.headers["Content-Type"] = writer.content_type
+      response.headers["Last-Modified"] = Time.now.httpdate # disable buffering middlewares
+
+      cursor = params.fetch(:start_index, 0).to_i
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      last_write = started
+
+      batch_limit = Xeno.config.stream_catch_up_batch
+      loop do
+        events = session.events.where(index: cursor..).order(:index).limit(batch_limit).to_a
+        events.each do |event|
+          writer.event(event)
+          cursor = event.index + 1
+        end
+        last_write = Process.clock_gettime(Process::CLOCK_MONOTONIC) if events.any?
+
+        # A full batch means more history is waiting — keep paging through
+        # the catch-up without sleeping or ending on a terminal status.
+        next if events.size == batch_limit
+
+        break unless session.reload.active?
+        break if stream_expired?(started)
+        # A graceful stop must not wait out the in-flight-request window —
+        # the stream is resumable by design (events are durable rows, the
+        # client reconnects with its cursor), so close it and let the
+        # server exit. The 5s force cap in the puma config stays as the
+        # backstop.
+        break if server_shutting_down?
+
+        # A quiet stream never writes, so a dead client would never raise and
+        # this thread would poll forever — keepalives make disconnects visible.
+        if Process.clock_gettime(Process::CLOCK_MONOTONIC) - last_write > KEEPALIVE_INTERVAL
+          writer.keepalive
+          last_write = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        sleep Xeno.config.stream_poll_interval
+      end
+    rescue ActionController::Live::ClientDisconnected, IOError
+      # the client went away — nothing to clean up, events are durable
+    ensure
+      writer&.close
+    end
+
+    private
+
+    def build_writer
+      if params[:format] == "ndjson" || request.headers["Accept"].to_s.include?("application/x-ndjson")
+        NdjsonWriter.new(response.stream)
+      else
+        SseWriter.new(response.stream)
+      end
+    end
+
+    def stream_expired?(started)
+      max = Xeno.config.stream_max_duration
+      max && Process.clock_gettime(Process::CLOCK_MONOTONIC) - started > max
+    end
+
+    def server_shutting_down?
+      check = Xeno.config.stream_shutdown_check
+      return !!instance_exec(&check) if check
+
+      !!puma_server&.shutting_down?
+    rescue StandardError
+      false
+    end
+
+    # ActionController::Live runs the action on its OWN thread, so Puma's
+    # thread-local Server.current is nil here (verified against a live
+    # server). Find the process's server instance once per connection.
+    def puma_server
+      return nil unless defined?(Puma::Server)
+
+      @puma_server ||= Puma::Server.current || ObjectSpace.each_object(Puma::Server).first
+    end
+  end
+end

data/app/helpers/xeno/application_helper.rb

@@ -0,0 +1,4 @@
+module Xeno
+  module ApplicationHelper
+  end
+end

data/app/jobs/xeno/application_job.rb

@@ -0,0 +1,4 @@
+module Xeno
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/xeno/reaper_job.rb

@@ -0,0 +1,12 @@
+module Xeno
+  # Periodic sweep for turns with no live owner (see Xeno::Reaper). Fired
+  # by Solid Queue's recurring machinery, a cron hitting `xeno:reap`, or
+  # any other scheduler — running it twice is harmless.
+  class ReaperJob < ApplicationJob
+    queue_as :default
+
+    def perform
+      Reaper.sweep!
+    end
+  end
+end

data/app/jobs/xeno/schedule_job.rb

@@ -0,0 +1,56 @@
+module Xeno
+  # Fires one scheduled run: a fire-and-forget task-mode session under the
+  # app principal. Solid Queue recurring entries (rake xeno:schedules:sync)
+  # and the dev dispatch endpoint both enqueue this.
+  #
+  # At-least-once safe: queues redeliver, and Solid Queue's recurring
+  # (task, run-at) dedup only covers the enqueue, not the execution. Each
+  # enqueue carries one ActiveJob job_id; the dedup row claims it in the
+  # SAME transaction that opens the session, so a redelivered tick can
+  # never double-open — it re-enqueues the recorded turn instead (idle
+  # no-op if the first delivery finished: the claim CAS decides).
+  class ScheduleJob < ApplicationJob
+    queue_as :default
+
+    discard_on ArgumentError
+
+    DEDUP_SCOPE = "schedule_run".freeze
+
+    def perform(name)
+      definition = Xeno.definition
+      schedule = definition.schedules[name]
+      raise ArgumentError, "unknown schedule: #{name}" unless schedule
+
+      turn = nil
+      claimed = Session.transaction do
+        Dedup.claim(DEDUP_SCOPE, job_id, metadata: { "schedule" => name }).tap do |row|
+          next unless row
+
+          session = Session.open!(
+            message: schedule.prompt,
+            channel: "schedule",
+            principal: { "type" => "app", "schedule" => name },
+            definition: definition
+          )
+          turn = session.turns.order(:sequence).last
+          row.update!(metadata: row.metadata.merge("session_id" => session.id, "turn_id" => turn.id))
+        end
+      end
+
+      turn ||= redelivered_turn(claimed)
+      TurnJob.perform_now(turn.id) if turn
+    end
+
+    private
+
+    # This job_id already ran (or died mid-run): pick up ITS turn instead of
+    # opening a second session. The claim CAS makes this a no-op when the
+    # first delivery actually finished.
+    def redelivered_turn(claimed)
+      return nil if claimed
+
+      turn_id = Dedup.existing(DEDUP_SCOPE, job_id)&.metadata&.fetch("turn_id", nil)
+      Turn.find_by(id: turn_id) if turn_id
+    end
+  end
+end

data/app/jobs/xeno/slack_event_job.rb

@@ -0,0 +1,20 @@
+module Xeno
+  # Processes one verified Slack event off the webhook path. The endpoint
+  # acks inside Slack's 3-second window and hands the payload here; session
+  # work (DB writes, LLM turns via TurnJob) happens on the queue. Slack
+  # retry storms die at the event_id dedup row (claimed atomically with
+  # this enqueue); a session can never be double-opened by a retry — the
+  # thread's continuation token is unique among active sessions.
+  class SlackEventJob < ApplicationJob
+    queue_as :default
+
+    discard_on ActiveJob::DeserializationError
+
+    def perform(payload)
+      channel = Channels.registry[:slack]
+      return unless channel
+
+      channel.handle_event(payload)
+    end
+  end
+end

data/app/jobs/xeno/turn_job.rb

@@ -0,0 +1,16 @@
+module Xeno
+  # One turn = one job. All correctness (claim, fencing, replay) lives in
+  # TurnRunner + the database; the queue only provides at-least-once
+  # delivery and retry backoff.
+  class TurnJob < ApplicationJob
+    queue_as :default
+
+    retry_on StandardError, wait: :polynomially_longer, attempts: 5
+    discard_on ActiveJob::DeserializationError, ActiveRecord::RecordNotFound
+
+    def perform(turn_id)
+      turn = Turn.find(turn_id)
+      TurnRunner.new(turn).run
+    end
+  end
+end

data/app/mailers/xeno/application_mailer.rb

@@ -0,0 +1,6 @@
+module Xeno
+  class ApplicationMailer < ActionMailer::Base
+    default from: "from@example.com"
+    layout "mailer"
+  end
+end

data/app/models/xeno/action.rb

@@ -0,0 +1,26 @@
+module Xeno
+  # One requested tool call: the durability checkpoint and (in M4) the
+  # approval state. A completed action means "executed AND its result is in
+  # the transcript" — the two are committed in one transaction, so replay
+  # logic can trust either signal.
+  class Action < ApplicationRecord
+    STATUSES = %w[pending pending_approval approved denied completed failed].freeze
+    KINDS = %w[tool question].freeze
+
+    belongs_to :turn, class_name: "Xeno::Turn"
+
+    validates :status, inclusion: { in: STATUSES }
+    validates :kind, inclusion: { in: KINDS }
+
+    def self.record!(turn, tool_call, kind: "tool")
+      find_or_create_by!(turn: turn, tool_call_id: tool_call.id) do |action|
+        action.tool_name = tool_call.name
+        action.kind = kind
+        action.input = tool_call.arguments
+      end
+    end
+
+    def completed? = status == "completed"
+    def awaiting_input? = %w[pending_approval].include?(status)
+  end
+end

data/app/models/xeno/application_record.rb

@@ -0,0 +1,5 @@
+module Xeno
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/xeno/chat.rb

@@ -0,0 +1,22 @@
+module Xeno
+  # The persisted RubyLLM chat backing one session's transcript.
+  # Owned by xeno — host apps with their own acts_as chat classes are
+  # unaffected; the message class below pins the transcript inside the
+  # engine namespace. Models, tool calls, usage, and batches are RubyLLM
+  # library records (ruby_llm_* tables), polymorphic toward this class.
+  class Chat < ApplicationRecord
+    acts_as_chat message_class: "Xeno::Message"
+
+    # assume_model_exists and protocol are runtime attributes, not columns —
+    # a freshly loaded record loses them, and the first to_llm build (which
+    # with_instructions triggers) then resolves the model strictly and with
+    # the provider's default protocol: ModelNotFoundError on the SECOND
+    # message of any session using a custom/OpenAI-compatible model
+    # (finding K). Every entry point re-applies them from the definition.
+    def apply_runtime_options!(model_options)
+      self.assume_model_exists = model_options[:assume_model_exists] if model_options.key?(:assume_model_exists)
+      self.protocol = model_options[:protocol] if model_options.key?(:protocol)
+      self
+    end
+  end
+end

data/app/models/xeno/dedup.rb

@@ -0,0 +1,24 @@
+module Xeno
+  # The at-least-once dedup ledger. Redelivered work (a cron tick fired
+  # twice, a Slack retry storm) claims its (scope, key) once; every other
+  # delivery loses the unique-index race and can read what the winner
+  # recorded in metadata.
+  class Dedup < ApplicationRecord
+    validates :scope, :key, presence: true
+
+    # Returns the freshly claimed row, or nil when the key was already
+    # claimed. Safe inside enclosing transactions (savepoint), safe under
+    # concurrency (the unique index decides).
+    def self.claim(scope, key, metadata: {})
+      transaction(requires_new: true) do
+        create!(scope: scope, key: key, metadata: metadata)
+      end
+    rescue ActiveRecord::RecordNotUnique
+      nil
+    end
+
+    def self.existing(scope, key)
+      find_by(scope: scope, key: key)
+    end
+  end
+end

data/app/models/xeno/event.rb

@@ -0,0 +1,63 @@
+module Xeno
+  # The append-only session stream — also the audit log. `index` is the
+  # per-session cursor; consumers dedupe
+  # re-emitted events by (session, index).
+  class Event < ApplicationRecord
+    belongs_to :session, class_name: "Xeno::Session"
+
+    # The fixed vocabulary (docs/runtime.md § Events). Hooks may subscribe
+    # to any of these or "*".
+    TYPES = %w[
+      session.started turn.started message.received
+      step.started step.completed step.failed
+      actions.requested action.result input.requested
+      reasoning.completed message.completed
+      compaction.requested compaction.completed
+      budget.exceeded
+      turn.completed turn.failed turn.cancelled
+      session.waiting session.completed session.failed
+    ].freeze
+
+    # Hooks fire only after the row is durably committed (and therefore
+    # visible to other processes) — never inside the emitting transaction.
+    after_create_commit { Hooks.dispatch(self) }
+
+    # Appends with a race-safe per-session index: concurrent writers collide
+    # on the unique index and retry with the next slot. The INSERT runs in
+    # its own savepoint (requires_new): emit is routinely called inside
+    # enclosing transactions, and on Postgres a failed INSERT otherwise
+    # aborts the whole transaction — the retry would raise
+    # PG::InFailedSqlTransaction instead of recovering.
+    def self.append!(session, event_type, data = {})
+      attempts = 0
+      begin
+        next_index = (where(session_id: session.id).maximum(:index) || -1) + 1
+        transaction(requires_new: true) do
+          create!(
+            session_id: session.id,
+            index: next_index,
+            event_type: event_type,
+            data: data,
+            created_at: Time.current
+          )
+        end
+      rescue ActiveRecord::RecordNotUnique
+        # Every collision means a competing append SUCCEEDED, so retries are
+        # bounded by actual contention. SQLite's file lock serializes writers
+        # (collisions are rare); Postgres runs them truly concurrently, so a
+        # burst can cost more than a handful of rounds — back off with jitter
+        # instead of giving up early.
+        attempts += 1
+        raise if attempts >= 50
+
+        sleep(rand * 0.002 * attempts)
+        retry
+      end
+    end
+
+    # The wire envelope: { type, data, meta: { index, at } }.
+    def envelope
+      { "type" => event_type, "data" => data || {}, "meta" => { "index" => index, "at" => created_at.iso8601(3) } }
+    end
+  end
+end

data/app/models/xeno/message.rb

@@ -0,0 +1,5 @@
+module Xeno
+  class Message < ApplicationRecord
+    acts_as_message chat_class: "Xeno::Chat"
+  end
+end

data/app/models/xeno/pending_message.rb

@@ -0,0 +1,7 @@
+module Xeno
+  # Messages that arrived while a turn was running. Drained into the next
+  # turn when the session parks or the turn completes.
+  class PendingMessage < ApplicationRecord
+    belongs_to :session, class_name: "Xeno::Session"
+  end
+end

data/app/models/xeno/session.rb

@@ -0,0 +1,231 @@
+module Xeno
+  # The durable conversation: lives for days or weeks, owns the transcript
+  # chat, the turn ledger, and the append-only event stream. Identified by
+  # id (inspect/stream handle) and at most one continuation_token (the
+  # channel-owned resume handle, unique among active sessions).
+  class Session < ApplicationRecord
+    STATUSES = %w[running waiting completed failed].freeze
+
+    belongs_to :chat, class_name: "Xeno::Chat"
+    has_many :turns, class_name: "Xeno::Turn", dependent: :destroy
+    has_many :events, class_name: "Xeno::Event", dependent: :destroy
+    has_many :pending_messages, class_name: "Xeno::PendingMessage", dependent: :destroy
+
+    validates :status, inclusion: { in: STATUSES }
+
+    # Opens a session for a first user message: builds the transcript chat
+    # per the resolved agent definition, stamps channel + principal, and
+    # stages turn 1. The caller enqueues the returned turn's job after the
+    # transaction commits (Session.start! does both).
+    def self.open!(message:, channel: nil, principal: nil, continuation_token: nil, definition: Xeno.definition)
+      transaction do
+        chat = Chat.new
+        options = definition.config.model_options
+        chat.apply_runtime_options!(options)
+        chat.provider = options[:provider].to_s if options[:provider]
+        chat.model = definition.config.resolved_model
+        chat.save!
+
+        session = create!(
+          agent: definition.name,
+          chat: chat,
+          channel: channel,
+          principal: principal,
+          continuation_token: continuation_token
+        )
+        session.emit("session.started", { agent: definition.name, channel: channel })
+        session.stage_turn!(message, definition: definition)
+        session
+      end
+    end
+
+    # Convenience for callers without transactional needs: open + enqueue.
+    def self.start!(message:, **options)
+      session = open!(message: message, **options)
+      session.turns.last.enqueue!
+      session
+    end
+
+    # Stages the next turn for one or more messages: persists each user
+    # message into the transcript, refreshes instructions (each turn picks up
+    # edited instructions without a restart), and
+    # appends the turn row. Runs in the caller's process so TurnJob replays
+    # never double-add the user message.
+    #
+    # defer_transcript: the turn row carries the messages but nothing is
+    # written into the transcript yet — the runner stages it when the turn
+    # is claimed. Required whenever an EARLIER turn may still be mid-flight
+    # (drained turns): appending user rows behind a parked turn's unanswered
+    # tool calls would wedge every later generate (providers demand tool
+    # results immediately after the assistant's tool_calls message).
+    def stage_turn!(messages, definition: Xeno.definition, emit_received: true, defer_transcript: false)
+      contents = Array(messages)
+      transaction do
+        unless defer_transcript
+          # Finding K: with_instructions triggers the first to_llm build; a
+          # fresh chat record must get its runtime options back first.
+          chat.apply_runtime_options!(definition.config.model_options)
+          resolved = definition.instructions_for(session: self)
+          chat.with_instructions(resolved) if resolved
+          contents.each { |content| chat.add_message(role: :user, content: content) }
+        end
+        turn = Turn.append!(self, user_message: { contents: contents }, transcript_deferred: defer_transcript)
+        if emit_received
+          contents.each { |content| emit("message.received", { content: content, turn_id: turn.id }) }
+        end
+        emit("turn.started", { turn_id: turn.id, sequence: turn.sequence })
+        turn
+      end
+    end
+
+    # The channel entry point for follow-up messages. While a turn is active
+    # (one active turn per session) the message queues in pending_messages
+    # and is folded into the next turn; a PARKED session drains immediately
+    # (the message becomes a visible staged turn instead of sitting invisible
+    # until someone resolves the input — mvp-design's delivery semantics).
+    # An idle session stages and enqueues a turn right away.
+    def receive_message!(content)
+      if turns.where(status: %w[pending running waiting]).exists?
+        transaction do
+          pending_messages.create!(payload: { "content" => content })
+          emit("message.received", { content: content, queued: true })
+        end
+        drain_pending_messages! if reload.status == "waiting"
+        nil
+      else
+        turn = stage_turn!(content)
+        turn.enqueue!
+        turn
+      end
+    end
+
+    # Folds every queued message into ONE next turn. Called when a turn
+    # parks or reaches a terminal status (and on messages to a parked
+    # session). Always defers the transcript: the drained turn may sit
+    # behind a parked turn whose tool calls are still unanswered, so its
+    # user rows are written only when the runner claims it — turn order and
+    # transcript order can never diverge. Row locks make a racing drain
+    # (park vs. incoming message) fold each message exactly once.
+    def drain_pending_messages!(definition: Xeno.definition)
+      turn = nil
+      transaction do
+        queued = pending_messages.order(:id).lock.to_a
+        next if queued.empty?
+
+        contents = queued.map { |m| m.payload["content"] }
+        turn = stage_turn!(contents, definition: definition, emit_received: false, defer_transcript: true)
+        queued.each(&:destroy!)
+      end
+      turn
+    end
+
+    # Steering (opt-in per message): stop what the agent is doing and make
+    # THIS message the next turn. The active turn dies safely — a parked or
+    # pending turn settles its dangling tool calls (recorded answers
+    # injected, unapproved gates denied — the H1 machinery) and cancels
+    # immediately; a running turn cancels cooperatively (the steer message
+    # queues and the runner's cancel path folds it into the next turn at the
+    # next step boundary). If the cancel loses the race with a completing
+    # turn, steering degrades to a normal follow-up — never an error.
+    def steer!(content, definition: Xeno.definition)
+      turn = turns.where(status: %w[pending running waiting]).order(:sequence).first
+
+      case turn&.status
+      when nil # idle: steer is just a message
+        staged = stage_turn!(content, definition: definition)
+        staged.enqueue!
+        staged
+      when "running"
+        transaction do
+          pending_messages.create!(payload: { "content" => content })
+          emit("message.received", { content: content, queued: true, steer: true })
+        end
+        chat.cancel! # cooperative; the runner settles, cancels, and drains
+        nil
+      else # pending or waiting — no process owns it; replace it now
+        transaction do
+          settle_unanswered_tool_calls!(turn, reason: "steered by user")
+          turn.update!(status: "cancelled")
+          emit("turn.cancelled", { turn_id: turn.id, steer: true })
+          update!(status: "running") if status == "waiting"
+        end
+        staged = stage_turn!(content, definition: definition)
+        staged.enqueue!
+        staged
+      end
+    end
+
+    # Stages a compaction turn (summarize-and-replace, executed by the
+    # runner under a normal claim). It queues behind any active or parked
+    # turn via the ordinary session-ordering machinery — the
+    # queued-behind-the-active-turn semantics — and never appends a
+    # synthetic user message.
+    def stage_compaction_turn!(reason:, used: nil, limit: nil)
+      transaction do
+        request = { "reason" => reason, "used" => used, "limit" => limit }.compact
+        turn = Turn.append!(self, kind: "compaction", user_message: { "compaction" => request })
+        emit("compaction.requested", { "turn_id" => turn.id }.merge(request))
+        emit("turn.started", { turn_id: turn.id, sequence: turn.sequence })
+        turn
+      end
+    end
+
+    def emit(event_type, data = {})
+      Event.append!(self, event_type, data)
+    end
+
+    # Called before a turn dies with tool calls still unanswered (cancel
+    # while parked, schedule park failure): every dangling call gets a tool
+    # result in the transcript. Without this the assistant tool_call rows
+    # stay unanswered forever and every later generate is a provider 400 —
+    # the session is bricked. A completed action (e.g. an answered question
+    # whose resume never ran) injects its recorded output; everything else
+    # gets a denial, recorded on the action row as the audit trail.
+    def settle_unanswered_tool_calls!(turn, reason:)
+      chat_record = Chat.find(chat_id)
+      llm = chat_record.to_llm
+      last_assistant = llm.messages.reverse.find { |m| m.role == :assistant }
+      return unless last_assistant&.tool_call?
+
+      answered = llm.messages.select { |m| m.role == :tool }.map(&:tool_call_id).compact.to_set
+      last_assistant.tool_calls.values.reject { |call| answered.include?(call.id) }.each do |call|
+        action = turn.actions.find_by(tool_call_id: call.id)
+        if action&.completed?
+          content = action.output&.fetch("content", nil).to_s
+          status = "completed"
+        else
+          content = JSON.generate({ denied: true, reason: reason })
+          status = "denied"
+        end
+
+        transaction do
+          action.update!(status: "denied", output: { "content" => content }) if action && status == "denied"
+          chat_record.add_message(role: :tool, content: content, tool_call_id: call.id)
+        end
+        emit("action.result", { turn_id: turn.id, call_id: call.id, tool: call.name, status: status })
+      end
+    end
+
+    # The session-scoped KV store (see Xeno::SessionState).
+    def state
+      @state_handle ||= SessionState.new(self)
+    end
+
+    def active?
+      %w[running waiting].include?(status)
+    end
+
+    # Terminal states release the continuation token (a finished session must
+    # not squat a channel thread's token); a copy stays in metadata for audit.
+    def finish!(new_status)
+      raise ArgumentError, "not a terminal status: #{new_status}" unless %w[completed failed].include?(new_status)
+
+      transaction do
+        audit = (metadata || {}).merge("released_continuation_token" => continuation_token)
+        # State is working memory for the conversation; reset clears it.
+        update!(status: new_status, continuation_token: nil, metadata: audit, state: {})
+        emit("session.#{new_status}")
+      end
+    end
+  end
+end