openlanes 3.0.0 → 3.1.0

data/lib/openlanes/live/copilot.rb

@@ -0,0 +1,290 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module OpenLanes
+  module Live
+    # The conversation with the run's copilot, c3po-operator (W17): one Redis
+    # stream per node ({Keys#copilot}) holding every turn, the person's and
+    # the copilot's, in the order they were said. The page reads it once with
+    # {#history} and then follows it on the event stream it already has
+    # (`event: copilot`), so the copilot needs no transport of its own.
+    #
+    # Who answers is the responder's business, not this class's: the API
+    # server hands it {Operator}, which runs c3po-operator, and `--demo` a
+    # canned line. Whatever answers writes {Turn}s, and {Turn.check} refuses
+    # a card that lacks what the page needs to draw it, whoever wrote it.
+    class Copilot
+      # A message the person may not send as written.
+      class Invalid < Error; end
+
+      # A message nobody is there to answer.
+      class Unanswered < Error; end
+
+      # Who spoke a turn.
+      SPEAKERS = %w[you copilot].freeze
+
+      # What a turn is: words, a red card asking to confirm a command that
+      # cannot be undone, or a quiet card holding a drafted answer.
+      KINDS = %w[text confirm draft].freeze
+
+      # The fields each card cannot be drawn without.
+      FIELDS = {
+        "text"    => [],
+        "confirm" => %w[command confirm done],
+        "draft"   => %w[question draft]
+      }.freeze
+
+      # Characters one message may carry. A longer one is a paste gone
+      # wrong, and the agent would pay for every token of it.
+      LIMIT = 4000
+
+      # Tokens and model before the copilot has said anything that counts them.
+      NO_USAGE = { model: nil, up: 0, down: 0 }.freeze
+
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param keys [OpenLanes::Live::Keys]
+      # @param responder [#call, nil] the person's {Turn} in, the copilot's
+      #   {Turn}s out; nil for a reader, which {#say} refuses to write through
+      def initialize(store:, keys:, responder: nil)
+        @store = store
+        @keys = keys
+        @responder = responder
+      end
+
+      # Record the person's message, and nothing more. Answering is
+      # {#answer}'s, run apart from the request that brought the message: a
+      # real responder is a `claude -p` call, and a POST held open for the
+      # whole of it would keep the page from showing the message, or that
+      # the copilot is reading, until the answer was already written.
+      #
+      # A slash command ({Slash}, plan 020.00) is not the copilot's to
+      # answer: it is queued for this node's engine under the message's id,
+      # and the engine's reply comes back into the conversation as turn
+      # `"<id>.1"`. A slash message that is not a command is answered here,
+      # at once, with the usage. Neither reaches the responder, so a server
+      # with no copilot still takes them, and {Turn#slash?} tells the caller
+      # not to {#answer} them.
+      #
+      # @param input [String, Hash] `CopilotMessage`: `{id, text}`
+      # @return [OpenLanes::Live::Copilot::Turn] the person's turn
+      # @raise [OpenLanes::Live::Copilot::Invalid] for the API server's 422
+      # @raise [OpenLanes::Live::Copilot::Unanswered] when nobody would
+      #   answer: no copilot for plain text, no live engine for a command
+      def say(input)
+        turn = message(input)
+        slash = Slash.parse(turn.text, id: turn.id)
+        # Refused before it is stored: a message kept where nothing will ever
+        # answer it reads, on the page, as a copilot that is merely slow.
+        raise Unanswered, "the copilot is not running in this server" unless slash || @responder
+        if slash.is_a?(Command) && !engine?
+          raise Unanswered, "no engine is running to apply #{turn.text.split.first}"
+        end
+
+        append(turn)
+        case slash
+        when Command then EventBus.for(@store).append(@keys.commands, { json: JSON.generate(slash.wire) })
+        when Slash::Help then append(Turn.new(id: "#{turn.id}.1", from: "copilot", kind: "text", text: slash.text))
+        end
+        turn
+      end
+
+      # Store whatever the responder answers to +turn+. A responder that
+      # fails is answered for, under the reply id the page waits for, so the
+      # failure shows in the conversation rather than as a page reading
+      # forever.
+      #
+      # @param turn [OpenLanes::Live::Copilot::Turn] the person's, from {#say}
+      # @return [Array<OpenLanes::Live::Copilot::Turn>] what was stored
+      def answer(turn)
+        replies = begin
+          @responder.call(turn)
+        rescue StandardError => e
+          [Turn.new(id: "#{turn.id}.1", from: "copilot", kind: "text", text: "c3po-operator could not answer: #{e.message}")]
+        end
+        replies.each { append(it) }
+      end
+
+      # The conversation as the page draws it on load: the newest turns,
+      # oldest first, and the copilot's latest model and tokens. The usage is
+      # looked for in the whole retained stream when the newest turns carry
+      # none, so a run of unmetered turns does not reset the header to zero.
+      #
+      # @param count [Integer]
+      # @return [Hash] `CopilotLog`: `{turns:, usage:}`
+      def history(count: 100)
+        turns = newest_turns(count)
+        usage = latest_usage(turns)
+        usage ||= latest_usage(newest_turns(Keys::COPILOT_LENGTH)) if turns.size == count
+        { turns:, usage: usage || NO_USAGE }
+      end
+
+      # Turns written after +after+, oldest first, with their stream ids.
+      #
+      # @param after [String] a stream id
+      # @return [Array<Array(String, Hash)>]
+      def since(after)
+        @store.xread(@keys.copilot, id: after).map { |entry| [entry.id, JSON.parse(entry.fields["json"])] }
+      end
+
+      # @return [String] the newest turn's stream id, `0-0` when there is none
+      def newest = @store.xrevrange(@keys.copilot, count: 1).first&.id || "0-0"
+
+      private
+
+      # @return [OpenLanes::Live::Copilot::Turn]
+      def message(input)
+        data = input.is_a?(String) ? JSON.parse(input) : input
+        raise Invalid, "a message is a JSON object" unless data.is_a?(Hash)
+
+        id = data["id"].to_s
+        text = data["text"].to_s.strip
+        raise Invalid, "id must be 1 to 64 letters, digits, - or _" unless id.match?(Command::ID)
+        raise Invalid, "a message needs text" if text.empty?
+        raise Invalid, "a message is at most #{LIMIT} characters" if text.length > LIMIT
+
+        Turn.new(id:, from: "you", kind: "text", text:)
+      rescue JSON::ParserError => e
+        raise Invalid, "not a message: #{e.message.lines.first.to_s.strip}"
+      end
+
+      # @return [Array<Hash>] up to +count+ turns, oldest first
+      def newest_turns(count)
+        @store.xrevrange(@keys.copilot, count:).reverse.map { JSON.parse(it.fields["json"]) }
+      end
+
+      # Whether this node's engine is there to read {Keys#commands}: its
+      # snapshot key lives only while it publishes (`EX 30`), so a command
+      # queued without one would sit unanswered and the page would read on.
+      #
+      # @return [Boolean]
+      def engine? = !@store.get(@keys.snapshot).nil?
+
+      # @return [Hash, nil]
+      def latest_usage(turns) = turns.reverse_each.map { it["usage"] }.find(&:itself)
+
+      # Written here, in the API server rather than by the Dispatcher, so the
+      # open streams are nudged here too: a turn has no other way to reach a
+      # stream that waits on {Keys#news}.
+      #
+      # @return [void]
+      def append(turn)
+        EventBus.for(@store).append(@keys.copilot, { json: turn.to_json }, maxlen: Keys::COPILOT_LENGTH)
+        News.publish(store: @store, keys: @keys, kind: "copilot")
+      end
+    end
+
+    class Copilot
+      # One thing said in the conversation: `CopilotTurn` in `types.ts`.
+      #
+      # @!attribute [r] id
+      #   @return [String] the person's message id; a reply's is that id, a
+      #     dot and its place among the replies, so a reply names what it answers
+      # @!attribute [r] from
+      #   @return [String] one of {SPEAKERS}
+      # @!attribute [r] kind
+      #   @return [String] one of {KINDS}
+      # @!attribute [r] text
+      #   @return [String]
+      # @!attribute [r] at
+      #   @return [Time]
+      # @!attribute [r] usage
+      #   @return [Hash, nil] `{model, up, down}`, the copilot's running total
+      #     after this turn; nil on the person's turns and on answers no model
+      #     wrote
+      # @!attribute [r] fields
+      #   @return [Hash{String => Object}] what a card needs, per {FIELDS}
+      Turn = Data.define(:id, :from, :kind, :text, :at, :usage, :fields) do
+        def initialize(at: Time.now, usage: nil, fields: {}, **rest)
+          super
+          Turn.check(self)
+        end
+
+        # @return [Hash] as the wire carries it
+        def wire = { id:, from:, kind:, text:, at: at.utc.iso8601, usage: }.merge(fields.transform_keys(&:to_sym))
+
+        # @return [Boolean] whether this is the person's slash message, which
+        #   the engine or {Slash} answers and the copilot never does
+        def slash? = from == "you" && text.start_with?("/")
+
+        # @return [String]
+        def to_json(*) = JSON.generate(wire)
+      end
+
+      class Turn
+        # The keys every turn has; anything else belongs to {FIELDS}.
+        BASE = %w[id from kind text at usage].freeze
+
+        # @param data [Hash] a turn as the wire carries it
+        # @return [OpenLanes::Live::Copilot::Turn]
+        # @raise [OpenLanes::Live::Copilot::Invalid]
+        def self.parse(data)
+          data = data.transform_keys(&:to_s)
+          new(id: data["id"].to_s,
+            from: data["from"].to_s,
+            kind: data["kind"].to_s,
+            text: data["text"].to_s,
+            at: data["at"] ? Time.iso8601(data["at"]) : Time.now,
+            usage: data["usage"],
+            fields: data.except(*BASE))
+        rescue ArgumentError => e
+          raise Invalid, "not a turn: #{e.message}"
+        end
+
+        # @param turn [OpenLanes::Live::Copilot::Turn]
+        # @return [void]
+        # @raise [OpenLanes::Live::Copilot::Invalid] naming what is wrong
+        def self.check(turn)
+          raise Invalid, "unknown speaker #{turn.from.inspect}" unless SPEAKERS.include?(turn.from)
+          raise Invalid, "unknown kind #{turn.kind.inspect}; known: #{KINDS.join(", ")}" unless KINDS.include?(turn.kind)
+          raise Invalid, "only the copilot draws cards" if turn.from == "you" && turn.kind != "text"
+
+          fields = turn.fields.transform_keys(&:to_s)
+          check_names(turn.kind, fields.keys)
+          check_text(turn.kind, fields)
+          check_command(fields["command"]) if turn.kind == "confirm"
+        end
+
+        # A card holds exactly its own fields: a missing one cannot be drawn,
+        # and an extra one could shadow a base key such as `id` on the wire.
+        #
+        # @return [void]
+        def self.check_names(kind, names)
+          wanted = FIELDS.fetch(kind)
+          missing = wanted - names
+          raise Invalid, "a #{kind} card needs #{missing.join(" and ")}" unless missing.empty?
+
+          extra = names - wanted
+          raise Invalid, "a #{kind} card has no #{extra.join(" or ")}" unless extra.empty?
+        end
+
+        # Every field but a command is drawn as text, so it has to be some:
+        # a blank one would draw an unnamed button or an empty draft.
+        #
+        # @return [void]
+        def self.check_text(kind, fields)
+          wrong = (FIELDS.fetch(kind) - %w[command]).reject { fields[it].is_a?(String) && !fields[it].strip.empty? }
+          raise Invalid, "a #{kind} card's #{wrong.join(" and ")} must be text" unless wrong.empty?
+        end
+
+        # A confirm card is sent through the same gate as the page's own
+        # buttons (W13), so it has to be a command that gate would take. It
+        # carries no id: the page mints one when the person presses, and an id
+        # supplied here would replace it and could repeat one already done.
+        #
+        # @return [void]
+        def self.check_command(command)
+          raise Invalid, "a confirm card's command is a JSON object" unless command.is_a?(Hash)
+
+          command = command.transform_keys(&:to_s)
+          raise Invalid, "a confirm card's command carries no id; the page mints one" if command.key?("id")
+
+          Command.parse(command.merge("id" => "card"))
+        rescue Command::Invalid => e
+          raise Invalid, "a confirm card's command is not one: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/openlanes/live/event_bus/base.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    module EventBus
+      # What every bus answers, delegated to the store behind it.
+      class Base
+        # @return [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+        attr_reader :store
+
+        # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+        def initialize(store)
+          @store = store
+          freeze
+        end
+
+        # @param channel [String]
+        # @param payload [String]
+        # @return [Integer] how many subscriptions received it
+        def publish(channel, payload) = store.publish(channel, payload)
+
+        # A subscription of its own; the caller closes it.
+        #
+        # @param channels [Array<String>]
+        # @param patterns [Array<String>]
+        # @return [#next_message, #close]
+        def subscribe(channels: [], patterns: []) = store.subscriber(channels:, patterns:)
+
+        # A durable append, in stream order.
+        #
+        # @param stream [String]
+        # @param fields [Hash]
+        # @param maxlen [Integer, nil] approximate trim
+        # @return [String] the entry id
+        def append(stream, fields, maxlen: nil) = store.xadd(stream, fields, maxlen:)
+
+        # The appends and writes the block makes, applied atomically.
+        #
+        # @yieldparam tx [OpenLanes::Live::EventBus::Transaction]
+        # @return [void]
+        def transaction
+          store.multi { |raw| yield Transaction.new(raw) }
+        end
+
+        # The database pub/sub channel names carry (see {Keys#news}).
+        #
+        # @return [Integer]
+        def db = store.db
+      end
+    end
+  end
+end

data/lib/openlanes/live/event_bus/memory_bus.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    module EventBus
+      # The bus for the suite and `--demo`, over {MemoryStore}: one process,
+      # one keyspace, the same contract.
+      class MemoryBus < Base
+        # @param store [OpenLanes::Live::MemoryStore]
+        def initialize(store = MemoryStore.new) = super
+      end
+    end
+  end
+end

data/lib/openlanes/live/event_bus/redis_bus.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    module EventBus
+      # The production bus, over {Store}: Redis pub/sub and streams.
+      class RedisBus < Base; end
+    end
+  end
+end

data/lib/openlanes/live/event_bus/transaction.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    module EventBus
+      # One {Base#transaction} block's writes: {#append} for the streams, and
+      # the store's other writes ({Store::WRITES}) beside them, so a site that
+      # sets a key and appends an entry together keeps doing both atomically.
+      class Transaction
+        # @param raw [OpenLanes::Live::Store::Transaction]
+        def initialize(raw)
+          @raw = raw
+        end
+
+        # @param stream [String]
+        # @param fields [Hash]
+        # @param maxlen [Integer, nil]
+        # @return [nil]
+        def append(stream, fields, maxlen: nil) = @raw.xadd(stream, fields, maxlen:)
+
+        (Store::WRITES - %i[xadd]).each do |name|
+          define_method(name) { |*args, **kwargs| @raw.public_send(name, *args, **kwargs) }
+        end
+      end
+    end
+  end
+end

data/lib/openlanes/live/event_bus.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    # The port every publish, subscribe and stream append goes through.
+    #
+    # Engine, Live and Web used to call `store.publish`, `store.subscriber`
+    # and `store.xadd` directly, on whichever of {Store} and {MemoryStore}
+    # they were handed, so the two shared a duck type nobody declared. A bus
+    # names that surface, and the stores become its backing: {RedisBus} over
+    # {Store}, {MemoryBus} over {MemoryStore}. Key layout, stream ids and the
+    # wire format are the store's, unchanged, because web cursors rely on them.
+    module EventBus
+      module_function
+
+      # The bus over +store+, or +store+ itself when it already is one.
+      #
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore, Base]
+      # @return [Base]
+      def for(store)
+        case store
+        when Base then store
+        when MemoryStore then MemoryBus.new(store)
+        else RedisBus.new(store)
+        end
+      end
+    end
+  end
+end

data/lib/openlanes/live/interviewer.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+module OpenLanes
+  module Live
+    # Grills a new idea before its spec.md is written (M6): the questions the
+    # agents downstream would otherwise block to ask, each as a short context
+    # paragraph and lettered options with exactly one recommended. The page
+    # asks them one at a time, and the answers reach {Commands#create_plans}
+    # as `payload.decisions`, which writes them into the spec.
+    #
+    # One call to a small model on OpenRouter, as {ChoiceConverter} makes,
+    # and every question is checked the same way: written with
+    # {Plans::Choices.write} and read back with {Plans::Choices.parse}. A
+    # question that does not survive the trip is dropped, not the interview:
+    # four good questions are worth asking without the fifth.
+    #
+    # A seam: the suite sets {Live.interviewer} to nil, and examples about it
+    # hand it an adapter double, so nothing in the tests reaches OpenRouter.
+    class Interviewer
+      # The most questions one idea is asked. A person is interviewed while
+      # they wait, so the list stays short (M0).
+      LIMIT = 5
+
+      # Seconds a person waits for the questions before the plan is created
+      # without them. The HTTP client alone allows 10 to connect and 30 to
+      # read, and a Puma thread and the dialog are held for all of it.
+      TIMEOUT = 20
+
+      # What the model reads with the idea.
+      PROMPT = <<~PROMPT.freeze
+        A person described a software feature they want built. Agents will turn it into a spec, a plan and
+        code without being able to ask anything, so interview the person now. Return `questions`: an array
+        of at most #{LIMIT} objects, the decisions that most change what gets built and that the idea leaves open.
+        Ask nothing the idea already answers. Each object is:
+
+        - `context`: one short paragraph saying what must be decided and why it matters.
+        - `options`: an array of 2 to 5 objects `{"label": String, "recommended": Boolean}`, each a distinct
+          answer on one short line. Exactly one has `"recommended": true`: the one you would choose.
+
+        Never add an option for "something else" or "type something instead": the page has a text box.
+        An idea that leaves nothing open gets an empty array.
+      PROMPT
+
+      # One question, ready for the page.
+      #
+      # @!attribute [r] context
+      #   @return [String]
+      # @!attribute [r] options
+      #   @return [Array<String>] labels, in order
+      # @!attribute [r] recommended
+      #   @return [Integer] the index of the recommended option
+      Question = Data.define(:context, :options, :recommended) do
+        # @return [Hash] as `POST /api/interview` sends it
+        def wire = { context:, options:, recommended: }
+      end
+
+      # An interviewer when OPENROUTER_API_KEY is set, nil otherwise.
+      #
+      # @param env [#[]]
+      # @param config [OpenLanes::Config]
+      # @return [OpenLanes::Live::Interviewer, nil]
+      def self.from_env(env = ENV, config: Config.current)
+        key = env["OPENROUTER_API_KEY"].to_s.strip
+        key.empty? ? nil : new(api_key: key, model: config.questions_model || ChoiceConverter::DEFAULT_MODEL)
+      end
+
+      # The model's questions, each cleaned the way {ChoiceConverter.render}
+      # cleans one, keeping those that survive.
+      #
+      # @param output [Hash] `questions`, as the adapter parsed it
+      # @return [Array<OpenLanes::Live::Interviewer::Question>] at most {LIMIT}
+      def self.render(output)
+        output = output.to_h.transform_keys(&:to_sym)
+        Array(output[:questions]).filter_map { question(it) }.first(LIMIT)
+      end
+
+      # @param raw [Object]
+      # @return [OpenLanes::Live::Interviewer::Question, nil]
+      def self.question(raw)
+        return unless raw.is_a?(Hash)
+
+        body = ChoiceConverter.render(raw)
+        choices = Plans::Choices.parse(body)
+        Question.new(context: choices.context,
+          options: choices.options.map(&:label),
+          recommended: choices.options.index(&:recommended))
+      rescue ChoiceConverter::Invalid
+        nil
+      end
+      private_class_method :question
+
+      # @param api_key [String, nil] OpenRouter's; only read to build the default adapter
+      # @param model [String] an OpenRouter model id
+      # @param adapter [#call, nil] answers `call(node, answers)` with a Hash
+      #   holding `questions`; built on first use when nil
+      # @param timeout [Numeric] seconds before the interview is given up
+      def initialize(api_key: nil, model: ChoiceConverter::DEFAULT_MODEL, adapter: nil, timeout: TIMEOUT)
+        @api_key = api_key
+        @model = model
+        @adapter = adapter
+        @timeout = timeout
+        @mutex = Mutex.new
+      end
+
+      # @return [String]
+      attr_reader :model
+
+      # Blocks for at most {TIMEOUT} seconds.
+      #
+      # @param idea [String] Markdown, as the idea dialog sends it
+      # @return [Array<OpenLanes::Live::Interviewer::Question>]
+      # @raise [StandardError] whatever the adapter raises: no network, a 4xx,
+      #   bad JSON, or Timeout::Error past the limit
+      def call(idea:)
+        output = Timeout.timeout(@timeout, Timeout::Error, "no questions within #{@timeout}s") do
+          adapter.call(node, { idea: idea.to_s })
+        end
+        self.class.render(output)
+      end
+
+      private
+
+      def adapter
+        @mutex.synchronize do
+          @adapter ||= begin
+            require "inquirex-llm"
+            Inquirex::LLM::OpenRouterAdapter.new(api_key: @api_key, model:)
+          end
+        end
+      end
+
+      def node
+        @node ||= begin
+          require "inquirex-llm"
+          Inquirex::LLM::Node.new(id: :interview,
+            verb: :extract,
+            prompt: PROMPT,
+            schema: Inquirex::LLM::Schema.new(questions: :array),
+            from_steps: %i[idea])
+        end
+      end
+    end
+  end
+end