insika 0.0.1 → 0.2.0

data/lib/insika/server/boot.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require "async"
+
+module Insika
+  module Server
+    # Turns the components into a service.
+    # MANDATORY order, no parallelism: plugins → stores → recovery → (app
+    # for the listen). "Never accepts a request before recovery" is guaranteed BY
+    # CONSTRUCTION: the listen (Falcon) only runs after `#call` returns the app, and
+    # `#call` only returns after `Recovery.run` finishes.
+    class Boot
+      # wiring: object with the named steps (load_plugins/build_stores/
+      # recovery/app) — the config/wiring.rb. logger: simple IO (default $stdout;
+      # nil silences). app: overrides the wiring's `app` step — for a serving arm
+      # (config.ru) that assembles its own Rack app around the wiring; the
+      # "recovery before the listen" guarantee is unchanged (#call still only
+      # returns after recovery).
+      def initialize(wiring, logger: $stdout, app: nil)
+        @wiring = wiring
+        @logger = logger
+        @app = app
+      end
+
+      # -> Rack app ready for the `run`. A store failure at boot (corrupted
+      # file → StoreError) PROPAGATES and aborts the process (coming up
+      # without durability is worse than not coming up); an unrecoverable task does NOT
+      # bring down the boot (Recovery already marks it :failed).
+      def call
+        @wiring.load_plugins
+        @wiring.build_stores
+        warn_if_ephemeral
+        summary = run_recovery
+        log("boot: recovery complete — #{summary[:resumed].size} resumed, " \
+            "#{summary[:failed].size} failed")
+        @app || @wiring.app
+      end
+
+      private
+
+      # Recovery dispatches resume_task, which creates task fibers — needs a
+      # current reactor. At config.ru load (Falcon) there is NO reactor: the Sync { }
+      # creates one and, by structured concurrency, only returns when the resume
+      # fibers FINISH (recovery + turns completed before the listen — slower
+      # boot, semantically safe). Under an already-current reactor (tests
+      # inside Async), it runs directly: returns after the resume DISPATCH, with
+      # the turns still in flight — also correct: "recovery before the listen" =
+      # dispatch before the listen, not turn completion.
+      def run_recovery
+        return do_recovery if Async::Task.current?
+
+        Sync { do_recovery }
+      end
+
+      # Task recovery THEN delegation recovery: the delegation
+      # sweep re-delivers completed-but-undelivered async delegations, and depends
+      # on the task sweep having re-dispatched any in-flight children first. Both
+      # create task fibers, so both must run inside the reactor scope of run_recovery.
+      #
+      # The TASK sweep is additionally gated per boot generation:
+      # its "orphaned :running" test cannot see a sibling worker's live fiber, so
+      # only the worker that claims the generation sweeps — the others would steal
+      # in-flight turns. The delegation and channel sweeps stay ungated: each of
+      # their records carries its own transactional claim (at-most-once holds
+      # however many workers sweep). Duck-typed: a wiring without the claim (test
+      # doubles, single-process arms) sweeps unconditionally.
+      def do_recovery
+        summary =
+          if skip_task_sweep?
+            log("boot: task sweep skipped — another worker claimed this boot generation")
+            { resumed: [], failed: [] }
+          else
+            @wiring.recovery.run
+          end
+        recover_delegations
+        recover_channel_deliveries
+        summary
+      end
+
+      def skip_task_sweep?
+        @wiring.respond_to?(:claim_recovery_sweep) && !@wiring.claim_recovery_sweep
+      end
+
+      # Duck-typed (like durable?): a wiring without async delegation just omits it.
+      def recover_delegations
+        return unless @wiring.respond_to?(:recover_delegations)
+
+        result = @wiring.recover_delegations
+        log("boot: delegations re-delivered — #{Array(result && result[:delivered]).size}")
+      end
+
+      # replies a previous process committed but never handed to the
+      # channel. Runs AFTER the task recovery for the same reason the delegation
+      # sweep does — a resumed turn writes its own outbox record at its terminal, and
+      # sweeping first would miss it.
+      def recover_channel_deliveries
+        return unless @wiring.respond_to?(:recover_channel_deliveries)
+
+        result = @wiring.recover_channel_deliveries
+        log("boot: channel replies re-dispatched — #{Array(result && result[:dispatched]).size}")
+      end
+
+      # Durability: without a durable backend, nothing is resumed after a
+      # restart — warns loudly at boot so we don't come up "without a net" by mistake. The
+      # test wiring (double) may not expose `durable?`; in that case, silence.
+      def warn_if_ephemeral
+        return unless @wiring.respond_to?(:durable?)
+        return if @wiring.durable?
+
+        log("boot: WARNING — EPHEMERAL backend (no INSIKA_DB): recovery will " \
+            "not resume anything after a restart.")
+      end
+
+      def log(message)
+        @logger&.puts(message)
+      end
+    end
+  end
+end

data/lib/insika/server/rack_app.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+# the /v1 transport as a VALUE the host app mounts, instead of a
+# server the engine starts.
+#
+#   mount Insika::Server.rack_app(INSIKA, token: ENV.fetch("INSIKA_TOKEN")), at: "/ai"
+#
+# The assembly used to live inline in DSL::ServerBoot#run, welded to
+# `Async::HTTP::Server.new(...).run` on the next line: a host that already owns a
+# reactor and a router could not reach it. Nothing here is new behavior — the
+# server boot now calls this instead of inlining it, which is what keeps the two
+# from drifting.
+#
+# The Studio is deliberately NOT part of this (the embed contract): it is
+# a class-level singleton, so it is one per process, and a host that wants the
+# operator UI mounts `Studio::App` itself and accepts that limitation.
+
+require_relative "app"
+
+module Insika
+  module Server
+    # `handle` is anything that answers #runtime (a DSL Definition/System) or a
+    # DSL::Runtime itself. -> a Rack app (`#call(env)`).
+    #
+    # It is mount-safe: routing reads `path_info`, so `Rack::URLMap`/Rails' `mount`
+    # moving the prefix into SCRIPT_NAME leaves every route intact.
+    def self.rack_app(handle, token: nil, **config)
+      AppBuilder.new(handle, token: token, **config).app
+    end
+
+    # Assembles Server::App from a graph. Also answers the two questions the boot
+    # banner asks (`workflows?`/`channels?`), so registering the env channels
+    # happens exactly once and in one place.
+    class AppBuilder
+      # Fixed local token: gates /v1 (Bearer) — and, under `serve`, logs into the
+      # Studio (cookie). Never a real secret; override with `token:`/ADMIN_TOKEN.
+      def initialize(handle, token: nil, **config)
+        @rt = handle.respond_to?(:runtime) ? handle.runtime : handle
+        @graph = @rt.graph
+        @token = token || ENV.fetch("ADMIN_TOKEN", "local-demo")
+        @config = config
+      end
+
+      attr_reader :token
+
+      def app
+        tenancy = @config[:tenancy] || ENV["INSIKA_TENANCY"] || "single_tenant"
+        # WS1: the token store is handed over ONLY in multi_tenant mode — in
+        # single_tenant the classic gateway token is the only credential
+        # (passing the store would silently widen the surface).
+        store = tenancy == "multi_tenant" ? @graph.token_store : nil
+        @app ||= Insika::Server::App.new(
+          command_bus: @graph.bus, event_stream: @graph.event_stream,
+          session_store: @graph.session_store, task_store: @graph.task_store,
+          pending_action_store: @graph.pending_action_store,
+          provisioner: Insika::PackImporter.new(bus: @graph.bus, profiles: @graph.profiles),
+          # GET /v1/agents/:id — the read-only capability view a case's `requires`
+          # resolves against.
+          profiles: @graph.profiles,
+          # the OSS onboarding surface (start.md + models.json + docs).
+          # This is the primary "build my first agent" target — models.json reports the
+          # DSL's stores + the agents this process serves (each id IS the `model`).
+          onboarding: build_onboarding,
+          # GET /v1/workflows + POST /v1/workflows/:name, opt-in by
+          # injection like every other edge — nil when the system declares none,
+          # so the routes simply do not exist (404, parity).
+          workflow_registry: (@graph.workflow_registry if workflows?),
+          # the bundled relay, when the env turns it on. Same rule as the
+          # OTEL bridge — a feature only `config.ru` can reach is a feature the
+          # docs are half-true about.
+          channels: (@graph.channel_registry if channels?),
+          config: { gateway_token: @token, tenancy: tenancy }.merge(@config),
+          token_store: store,
+          # a 500's error_ref must be findable in the process log.
+          logger: $stdout
+        )
+      end
+
+      def workflows? = !@graph.workflow_registry.names.empty?
+
+      # Registers the env-configured channels once, and reports whether any exist.
+      def channels?
+        unless defined?(@channels_ready)
+          @channels_ready = true
+          relay = Insika::Channels::Relay.from_env(
+            http: Insika::HttpClient.new,
+            allow_http: Insika::EnvSchema.truthy?(ENV["INSIKA_EGRESS_ALLOW_HTTP"]),
+            allow_private: Insika::EnvSchema.truthy?(ENV["INSIKA_EGRESS_ALLOW_PRIVATE"])
+          )
+          @graph.channel_registry.register(relay.id, relay) if relay
+
+          widget = Insika::Channels::Web.from_env(
+            chat_rate_limit: Insika::Channels::Web.limit_resolver(
+              profiles: @graph.profiles, settings_store: @rt.component(:settings_store)
+            )
+          )
+          @graph.channel_registry.register(widget.id, widget) if widget
+        end
+        !@graph.channel_registry.names.empty?
+      end
+
+      private
+
+      def build_onboarding
+        configs = @rt.packs.map(&:config)
+        Insika::Onboarding.standard(
+          root: File.expand_path("../../..", __dir__),
+          settings_store: @rt.component(:settings_store),
+          provider_store: @rt.component(:provider_store),
+          # EVERY agent this process serves — each id IS a `model` on
+          # /v1/responses, so a coding agent reading models.json sees the whole
+          # system, not just the first one.
+          agents: -> { configs.map { |c| { id: c[:id], model: c[:model], provider: c[:provider] } } }
+        )
+      end
+    end
+  end
+end

data/lib/insika/server/responses.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  module Server
+    # OpenAI Responses edge adapter (`/v1/responses`) — the contract that
+    # OpenClaw gateway consumers already speak.
+    #
+    # PURE module (no state, no framework): (a) translates the OpenAI
+    # Responses request → `:send_message` payload; (b) maps each turn Event →
+    # OpenAI Responses SSE frame (or nil for events with no counterpart). Follows the
+    # constitutional rule: no business logic, no store access here.
+    #
+    # Request: { model: "openclaw:<agent>", user: "<chat.id>", stream: true,
+    #            input: "<string with already-composed blocks>" } + header
+    # X-Openclaw-Agent (agent fallback). The `input` enters VERBATIM as the
+    # turn's message — the blocks (<memoria>/<dados_conhecidos>/directives) already come
+    # composed by the consumer (the engine does not interpret them).
+    module Responses
+      module_function
+
+      # -> { agent:, user:, message:, origin? } | raise ValidationError.
+      #
+      # `origin` is the consumer declaring WHO wrote the input it is sending. It
+      # matters here more than anywhere: this adapter's `input` is a STRING the
+      # consumer already composed out of context blocks plus the customer's text
+      # (`<memoria> …`, `<store_cep_required> …`), so a transcript reader cannot
+      # tell the two apart — the first refinement run over real traffic reported 219
+      # "the customer repeated themselves" that were the engine reading its own
+      # fragment back. A consumer that sends `origin: "engine"` on a composed turn
+      # gets that filtered structurally instead of by a regex on the leading tag.
+      # Omitted = a customer typed it, which is what every turn meant before.
+      def parse_request(body, req)
+        agent = body[:model].to_s.sub(/\Aopenclaw:/, "")
+        agent = req.get_header("HTTP_X_OPENCLAW_AGENT").to_s if agent.empty?
+        raise Insika::ValidationError, "model/agent missing" if agent.strip.empty?
+
+        user = body[:user].to_s
+        raise Insika::ValidationError, "user missing" if user.strip.empty?
+
+        message = extract_input(body[:input])
+        raise Insika::ValidationError, "input empty" if message.strip.empty?
+
+        out = { agent: agent.strip, user: user, message: message }
+        (origin = Insika::MessageOrigin.parse!(body[:origin])) && (out[:origin] = origin)
+        out
+      end
+
+      # V1: `input` is a STRING (the dispatcher composes the blocks + user text). Tolerates
+      # an array of parts (OpenAI multimodal shape) by joining the texts.
+      def extract_input(input)
+        case input
+        when String then input
+        when Array
+          input.flat_map { |part| part.is_a?(Hash) ? (part[:text] || part["text"]) : part }
+               .compact.join("\n")
+        else input.to_s
+        end
+      end
+
+      # Turn Event -> OpenAI Responses SSE frame | nil (event with no
+      # counterpart: :task_started, :tool_result, :skill_activated, ...).
+      # Terminal events emit the final frame + `[DONE]` (close the stream).
+      def frame_for(event)
+        case event.type
+        when :content
+          sse("response.output_text.delta",
+              { type: "response.output_text.delta", delta: event.data[:delta].to_s })
+        when :tool_call
+          sse("response.output_item.added",
+              { type: "response.output_item.added",
+                item: { type: "function_call", name: event.data[:name].to_s } })
+        when :task_completed
+          completed(event) + done
+        when :task_failed
+          failed(event.data[:message] || "task failed") + done
+        when :task_cancelled
+          failed("task cancelled") + done
+        when :error
+          failed(event.data[:message] || "error") + done
+        when :thinking
+          # The provider's reasoning. Internal unless the AGENT opted in
+          # (`edge_stream thinking: true`), which tags the event. Even then it does
+          # NOT become answer text: it gets the Responses reasoning frame, so a
+          # consumer that only accumulates `output_text` deltas — a dispatcher
+          # that turns them into one WhatsApp message — is unaffected,
+          # and one that renders reasoning has something to render.
+          if public_delta(event)
+            sse("response.reasoning_summary_text.delta",
+                { type: "response.reasoning_summary_text.delta", delta: event.data[:delta].to_s })
+          end
+        when :intermediate
+          # The model's own prose that did not turn out to be the answer — the
+          # narration of a message that also called a tool, or the reasoning-in-content
+          # a model emits when it has no tool to call. A real store's prompt sent 132
+          # deltas of an English monologue this way before TurnOutput held them back.
+          #
+          # NAMESPACED on purpose when published. There is no `response.*` event for
+          # "text the assistant said that is not the answer": in the real protocol that
+          # text IS `output_text.delta`, told apart only by an output-item index this
+          # adapter does not carry. So a `response.*` type here would be a lie a strict
+          # client would believe. `insika.*` is obviously ours and unknown types are
+          # ignored — which is the safe failure.
+          if public_delta(event)
+            sse("insika.intermediate.delta",
+                { type: "insika.intermediate.delta", delta: event.data[:delta].to_s })
+          end
+        when :guardrail_blocked, :guardrail_flagged
+          # audit events with no OpenAI Responses counterpart. On a BLOCK
+          # the safe reply still reaches the consumer through the normal :content
+          # deltas + :task_completed path (the turn completes gracefully), so there
+          # is nothing extra to translate here — the events live in /v1/events + the
+          # Studio + the trace. Explicit (not a fall-through) to keep the closed
+          # catalog honest.
+          nil
+        when :ttft
+          # the live TTFB signal (WS6, INSIKA_TURN_TIMING opt-in): the provider's
+          # ms-to-first-token, emitted when the first content chunk arrives.
+          # Namespaced insika.* — no OpenAI Responses counterpart; unknown types
+          # are ignored, the safe failure.
+          sse("insika.ttft", { type: "insika.ttft", ttft_ms: event.data[:ttft_ms].to_i })
+        end
+      end
+
+      # Did the AGENT opt this channel in? The Executor tags the event (`edge_stream`)
+      # because this mapper is pure and static — no agent, no stores, no state. An
+      # untagged event is internal, which is the default and the safe reading — and
+      # "not published" must be nil, like every other unmapped event in the catalog.
+      def public_delta(event) = event.data[:public] == true
+
+      def completed(event)
+        response = {}
+        if (usage = event.data[:usage])
+          # `model` travels alongside usage in the event; in the OpenAI shape it is a sibling of
+          # usage (pure tokens in usage).
+          model = usage[:model] || usage["model"]
+          response[:usage] = usage.reject { |k, _| k.to_s == "model" }
+          response[:model] = model if model
+        end
+        # Opt-in per-turn latency breakdown (INSIKA_TURN_TIMING). Absent
+        # by default — a non-standard sibling used only for TTFB diagnostics.
+        (timing = event.data[:timing]) && (response[:timing] = timing)
+        # WS5 stuck signal: an additive sibling the terminal frame carries when the
+        # agent ended the turn declaring it cannot proceed. Consumers that only read
+        # the OpenAI-shaped response.use it to run their escalation ("stuck" means
+        # what they decide it means, never the engine's business).
+        (outcome = event.data[:outcome]) && (response[:outcome] = outcome.to_s)
+        sse("response.completed", { type: "response.completed", response: response })
+      end
+
+      def failed(message)
+        sse("response.failed",
+            { type: "response.failed", response: { error: { message: message.to_s } } })
+      end
+
+      # event: + data: (the dispatcher reads both: `event:` and `type` in the JSON).
+      def sse(event_name, data)
+        "event: #{event_name}\ndata: #{JSON.generate(data)}\n\n"
+      end
+
+      def done = "data: [DONE]\n\n"
+    end
+  end
+end

data/lib/insika/server/sse_body.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "json"
+require "timeout"
+require "async"
+require "async/queue"
+
+module Insika
+  module Server
+    # SSE response body (evolves `SSEStream`). `SSEStream`
+    # received a producer block (the Runner wrote into it); `SSEBody` DRAINS a
+    # `Subscription` from the EventStream: each subscriber has its
+    # own queue; `#each` blocks the CONSUMER's fiber until the subscription
+    # closes. The wire is EXACTLY `Event#to_h`.
+    class SSEBody
+      PING = ": ping\n\n" # SSE comment — doesn't pollute the consumer
+
+      # serialize: maps an Event -> String (SSE frame) OR nil (discarded
+      # event, no frame). Default = the canonical wire `data: <Event#to_h>`.
+      # The /v1/responses adapter injects a serializer that produces OpenAI
+      # Responses events (and skips those with no counterpart).
+      DEFAULT_SERIALIZE = ->(event) { "data: #{JSON.generate(event.to_h)}\n\n" }
+
+      # subscription: any object with #each (yields Events) and #close.
+      # heartbeat: seconds of silence before emitting a ping (15s
+      # clears 60s ALB/nginx idle timeouts with room to spare).
+      def initialize(subscription:, heartbeat: 15, serialize: nil)
+        @subscription = subscription
+        @heartbeat = heartbeat
+        @serialize = serialize || DEFAULT_SERIALIZE
+      end
+
+      # Rack 3 STREAMING BODY (`#call(stream)`), NOT `#each`. Under
+      # protocol-rack/protocol-http1 (the stack of Async::HTTP::Server AND Falcon),
+      # a body that responds to `#each` is routed to Body::Enumerable, whose
+      # `read` runs the `#each` in a PLAIN Enumerator Fiber (not an Async::Task) —
+      # there `Async::Task.current` raises "No async task available", the loop died
+      # swallowed in the rescue and the body came out EMPTY. By exposing `#call` (and NOT `#each`),
+      # the body is routed to Body::Streaming, which schedules the block via
+      # Fiber.schedule under the reactor's scheduler — so the subscription drains and the
+      # frames actually reach the socket (incrementally).
+      #
+      # The `stream` (Protocol::HTTP::Body::Stream) responds to #write/#close. Drains the
+      # subscription DIRECTLY (no Async::Task.current): when this fiber blocks
+      # waiting for the next event, the scheduler runs the writer, which pushes the
+      # already-written frame to the socket. Heartbeat via Timeout.timeout (scheduler hook),
+      # which works in the scheduled fiber — keeps the connection alive while idle (L4).
+      def call(stream)
+        drain(stream)
+      rescue StandardError
+        # Client disconnected: `stream.write` raises when the socket closes.
+        # No exception escapes; the turn's task is NEVER cancelled here — the
+        # execution belongs to the runtime, not the connection (reconnect at /v1/events).
+        nil
+      ensure
+        @subscription.close
+        stream.close
+      end
+
+      private
+
+      def drain(stream)
+        internal = Async::Queue.new
+        closed = Object.new # end-of-subscription sentinel
+
+        # Child fiber (scheduled on the reactor's scheduler): drains the subscription
+        # into an internal queue and, on close, pushes the sentinel. Isolates the
+        # subscription's blocking from the heartbeat loop. Ends on its own when
+        # `@subscription.close` (in #call's ensure) makes the `each` finish — no
+        # need to kill the fiber by hand.
+        Fiber.schedule do
+          @subscription.each { |event| internal.enqueue(event) }
+        ensure
+          internal.enqueue(closed)
+        end
+
+        loop do
+          event =
+            begin
+              Timeout.timeout(@heartbeat) { internal.dequeue }
+            rescue Timeout::Error
+              :heartbeat # no event within `heartbeat`s -> ping
+            end
+
+          if event.equal?(:heartbeat)
+            stream.write(PING)
+          elsif event.equal?(closed)
+            break
+          elsif (frame = @serialize.call(event))
+            stream.write(frame)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/insika/server/tenant_auth.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "rack"
+require "rack/utils"
+
+module Insika
+  module Server
+    # Edge resolution for WS1 (multi-tenant): `Authorization: Bearer <token>` ->
+    # a principal `{ role:, tenant_id: }`, resolved BEFORE the routes. Two modes,
+    # one gate:
+    #
+    #   single_tenant (default) — no token store: the classic single operator
+    #     credential (config[:gateway_token]) is the only thing that resolves.
+    #   multi_tenant — tokens live in the TokenStore (per-tenant + operator).
+    #     A configured gateway_token STILL resolves as operator (an existing
+    #     deployment switching modes keeps its credential — additive, never
+    #     a second-class path).
+    #
+    # Fail-closed BY CONSTRUCTION: no store and no configured token -> :disabled
+    # (503, never open). A revoked or unknown token -> :unauthorized. Pure module,
+    # testable without a Rack env.
+    module TenantAuth
+      module_function
+
+      # gateway_token: config[:gateway_token] | nil. token_store: TokenStore |
+      # nil. header: raw Authorization value.
+      # -> :disabled | :unauthorized | { role: "operator"|"tenant", tenant_id: }
+      def check(gateway_token, token_store, header)
+        # Fail-closed FIRST (the construction rule): with no store AND no
+        # configured token the gateway is DISABLED (503) however the request
+        # looks — never "401: who are you facing a door that does not exist".
+        # A token_store present means the gateway IS configured (multi_tenant),
+        # with or without the legacy gateway token.
+        return :disabled if token_store.nil? && (gateway_token.nil? || gateway_token.empty?)
+
+        provided = header.to_s[/\ABearer (.+)\z/, 1]
+        return :unauthorized if provided.nil?
+
+        if token_store
+          record = token_store.resolve(provided)
+          unless record
+            # store miss -> the legacy gateway token still resolves as operator
+            # (an existing deployment switching modes keeps its credential).
+            return :unauthorized if gateway_token.nil? || gateway_token.empty?
+            return :unauthorized unless Rack::Utils.secure_compare(gateway_token, provided)
+
+            return { role: "operator", tenant_id: nil }
+          end
+
+          return { role: record.role.to_s, tenant_id: record.tenant_id }
+        end
+
+        # classic mode (no store): the gateway token is the only credential.
+        # Constant-time comparison: the operator token doesn't leak via timing.
+        return :unauthorized unless Rack::Utils.secure_compare(gateway_token, provided)
+
+        { role: "operator", tenant_id: nil }
+      end
+    end
+  end
+end