insika 0.0.1 → 0.2.0

data/lib/insika/config_store.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  # CONFIGURATION DOMAIN store. Unlike the
+  # EXECUTION stores (session/task/checkpoint/pending/memory), this holds the
+  # configuration the Studio authors at runtime: agents (profiles), general
+  # settings, LLM providers and MCP instances. Scoped KV over any
+  # `Insika::Store` — durable when the backend is SQLite (survives a restart,
+  # like everything else).
+  #
+  # logical scope -> physical namespace "config:<scope>". Values are
+  # JSON-serializable Hashes (symbol becomes string on the round-trip, per the
+  # Store contract; the domain re-symbolizes at the edge — see StoredProfileSource).
+  class ConfigStore
+    SCOPE_PREFIX = "config"
+    # agent_files/skills: the CONTENT of per-agent prompts
+    # and shared skills lives in the Store (single source of truth,
+    # a SQLite backup), not on disk — disk becomes only seed/import.
+    # goldens: authored eval cases — config, like the
+    # prompts and skills: the corpus on disk is seed and export, the store is what a
+    # deployment runs and what the Studio edits.
+    # baselines: the ACCEPTED state of each agent's golden set.
+    # Config for the same reason: it is a curated decision ("this is the bar"), the
+    # file is its export, and the refinement gate reads it from inside a deployment
+    # that has no checkout.
+    # agent_skills: the per-agent SPECIALIZATIONS of a shared skill (and
+    # agent-private skills). A second scope rather than a composite key in `skills`,
+    # so the shared records are untouched by the agent dimension arriving — no
+    # migration, and a live deployment keeps serving exactly what it served.
+    SCOPES = %w[agents settings llm_providers mcp agent_files skills agent_skills
+                system_files tools goldens baselines].freeze
+
+    class UnknownScope < Insika::Error; end
+
+    def initialize(store:)
+      @store = store
+    end
+
+    # Upsert (last-write-wins). -> value (the same Hash that was passed)
+    def put(scope, key, value)
+      @store.set(ns(scope), key.to_s, stringify(value))
+      value
+    end
+
+    # -> Hash | nil
+    def get(scope, key)
+      @store.get(ns(scope), key.to_s)
+    end
+
+    # -> bool (did it exist?)
+    def delete(scope, key)
+      @store.delete(ns(scope), key.to_s)
+    end
+
+    # -> [String] keys of the scope, sorted lexicographically
+    def keys(scope)
+      @store.list(ns(scope))
+    end
+
+    # -> [Hash] all records of the scope (lexicographic key order)
+    def all(scope)
+      s = ns(scope)
+      @store.list(s).filter_map { |k| @store.get(s, k) }
+    end
+
+    private
+
+    def ns(scope)
+      key = scope.to_s
+      raise UnknownScope, "unknown config scope: #{scope.inspect}" unless SCOPES.include?(key)
+
+      "#{SCOPE_PREFIX}:#{key}"
+    end
+
+    # Normalizes symbol->string BEFORE writing (mirrors MemoryStore#stringify):
+    # the backend serializes JSON, and reading back would return strings anyway;
+    # normalizing on write keeps the record consistent across backends.
+    def stringify(obj)
+      case obj
+      when Hash then obj.each_with_object({}) { |(k, v), acc| acc[k.to_s] = stringify(v) }
+      when Array then obj.map { |v| stringify(v) }
+      when Symbol then obj.to_s
+      else obj
+      end
+    end
+  end
+end

data/lib/insika/context/builder.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require "async"
+require "time"
+
+module Insika
+  # Builder output, consumed by the Executor in stage 5.
+  #   system:       String (final concatenation for with_instructions)
+  #   history:      [{role:, content:}] (for seeding the chat)
+  #   tool_context: String | nil
+  #   fragments:    [ContextFragment] post-cut, in canonical order (audit)
+  #   budget:       { cap:, used:, evicted: [source] }
+  ContextPackage = Data.define(:system, :history, :tool_context, :fragments, :budget)
+
+  # Stage 2 of the pipeline: the Runtime NEVER builds the prompt — it asks the
+  # Builder for the package. Implements selection -> fan-out production ->
+  # collection/estimation -> budgeting with eviction -> canonical assembly.
+  class ContextBuilder
+    def initialize(providers:, event_stream:, hooks: Hooks.new, estimator: TokenEstimator)
+      @providers = providers
+      @event_stream = event_stream
+      @hooks = hooks # prompt pair; empty Hooks = no-op
+      @estimator = estimator
+    end
+
+    # The :prompt pair is wrapped HERE, not in the Executor: before_prompt
+    # can rewrite the ContextRequest (providers run with the modified one);
+    # after_prompt can rewrite the assembled ContextPackage. IMPORTANT: the
+    # Executor calls only `builder.call(request)` — do NOT wrap it with
+    # around(:prompt) again (double-wrapping would fire the hooks twice).
+    def call(request)
+      @hooks.around(:prompt, request) { |req| build_package(req) }
+    end
+
+    private
+
+    def build_package(request)
+      selected = select_providers(request.profile)
+      fragments = estimate_tokens(produce(selected, request))
+      cap = request.profile.limits[:context_budget] || 8_000
+      fragments, evicted = apply_budget(fragments, cap)
+      unless evicted.empty?
+        emit_warning("ContextBuilder",
+                     "budget: #{evicted.size} fragment(s) evicted from #{evicted.uniq}",
+                     request)
+      end
+      assemble(fragments, cap, evicted)
+    end
+
+    # Step 1: selection — enabled_for? AND the profile allowlist.
+    def select_providers(profile)
+      @providers.select do |p|
+        p.enabled_for?(profile) && Allowlist.allows?(profile.context_providers, p.id)
+      end
+    end
+
+    # Step 2: fan-out production with a BARRIER and a per-provider timeout
+    # (with_timeout — never Timeout.timeout). Each provider is a CHILD fiber
+    # of the current fiber: cancelling the task cancels the in-flight production.
+    def produce(selected, request)
+      timeout = request.profile.limits[:provider_timeout] || 5
+      tasks = selected.map do |provider|
+        child = Async::Task.current.async { |t| t.with_timeout(timeout) { provider.call(request) } }
+        [provider, child]
+      end
+
+      fragments = []
+      tasks.each do |provider, child|
+        fragments.concat(Array(child.wait))
+      rescue StandardError => e # Async::TimeoutError is a StandardError; Async::Stop is NOT (propagates)
+        handle_provider_failure(provider, e, request)
+      end
+      fragments
+    end
+
+    # A required provider failing -> ContextError
+    # (aborts the turn, mapped by the Executor); an optional one failing ->
+    # warning + graceful degradation (fragments omitted, turn proceeds).
+    def handle_provider_failure(provider, error, request)
+      if provider.required?
+        raise ContextError.new("required provider '#{provider.id}' failed: #{error.message}",
+                               provider: provider.id)
+      end
+
+      emit_warning(provider.id, error.message, request)
+    end
+
+    # Step 3: estimate tokens only when the provider did not report them.
+    def estimate_tokens(fragments)
+      fragments.map { |f| f.tokens ? f : f.with(tokens: @estimator.estimate(estimable_text(f.content))) }
+    end
+
+    # History fragments carry a Hash {role:, content:} as their content;
+    # estimating over the Hash would count the `#to_s` text (":role=>", quotes,
+    # symbols), inflating each message and biasing the eviction. Count the values,
+    # not the Ruby representation.
+    def estimable_text(content)
+      case content
+      when String then content
+      # An eviction unit (R1): a cycle of message Hashes -> sum their text.
+      when Array then content.map { |c| estimable_text(c) }.join(" ")
+      when Hash then content.values.map(&:to_s).join(" ")
+      else content.to_s
+      end
+    end
+
+    # Steps 4-5: GLOBAL budget. Cuts non-pinned fragments from lowest priority to
+    # highest; ties -> lowest production index first (stable cut: among
+    # histories, the oldest drops first). pinned is uncuttable; if pinned alone
+    # already exceeds -> ContextError (do not truncate identity).
+    def apply_budget(fragments, cap)
+      used = fragments.sum(&:tokens)
+      return [fragments, []] if used <= cap
+
+      indexed = fragments.each_with_index.to_a
+      cuttable = indexed.reject { |f, _i| f.pinned }.sort_by { |f, i| [f.priority, i] }
+      evicted_idx = []
+      evicted_sources = []
+      cuttable.each do |fragment, index|
+        break if used <= cap
+
+        used -= fragment.tokens
+        evicted_sources << fragment.source
+        evicted_idx << index
+      end
+
+      if used > cap
+        raise ContextError.new(
+          "unsolvable budget: pinned fragments (#{used} tokens) exceed the cap (#{cap})",
+          provider: "ContextBuilder"
+        )
+      end
+
+      survivors = fragments.each_index.reject { |i| evicted_idx.include?(i) }.map { |i| fragments[i] }
+      [survivors, evicted_sources]
+    end
+
+    # Step 6: assembly in DETERMINISTIC canonical order.
+    def assemble(fragments, cap, evicted)
+      system_frags = fragments.select { |f| f.placement == :system }
+                              .sort_by.with_index { |f, i| [-f.priority, f.source.to_s, i] }
+      history_frags = fragments.select { |f| f.placement == :history } # production order (chronological)
+      tool_frags = fragments.select { |f| f.placement == :tool_context }
+
+      system = system_frags.map(&:content).join("\n\n")
+      history = history_frags.map(&:content)
+      tool_context = tool_frags.empty? ? nil : tool_frags.map(&:content).join("\n\n")
+
+      canonical = system_frags + history_frags + tool_frags
+      ContextPackage.new(
+        system: system, history: history, tool_context: tool_context,
+        fragments: canonical, budget: { cap: cap, used: canonical.sum(&:tokens), evicted: evicted }
+      )
+    end
+
+    # :provider_warning. The Builder does not know task_id/seq (correlation is
+    # the Executor's job) — emits with what it has; Event#to_h does meta.compact.
+    def emit_warning(provider_id, message, request)
+      @event_stream.emit(Insika::Event.new(
+                           type: :provider_warning,
+                           data: { provider: provider_id, message: message },
+                           meta: { session_id: request.session&.id, at: Time.now.utc.iso8601 }
+                         ))
+    end
+  end
+end

data/lib/insika/context/catalog_provider.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Insika
+  module Context
+    # Base for providers that inject level 1 of a catalog (skills, deferred
+    # tools) as a :system fragment. The subclass says WHICH entries and the
+    # priority; the fragment assembly (format -> skip if empty -> a single
+    # fragment with source = the provider id) is shared.
+    class CatalogProvider < ContextProvider
+      def initialize(catalog:)
+        @catalog = catalog
+      end
+
+      def call(request)
+        block = @catalog.format_for_prompt(entries(request))
+        return [] if block.empty?
+
+        [ContextFragment.build(content: block, placement: :system,
+                               priority: priority, source: id)]
+      end
+    end
+  end
+end

data/lib/insika/context/fragment.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Insika
+  # Unit of context produced by a provider.
+  # SHARED type (Insika::, not Insika::Context::).
+  #   placement: :system | :history | :tool_context
+  #   priority:  Integer; higher = more important (survives cuts)
+  #   tokens:    Integer | nil; estimated by the Builder when nil
+  #   source:    String — provider id (audit)
+  #   pinned:    true -> uncuttable in the budget (e.g. identity)
+  #   labels:    [{ "name" =>, "reason" => }] — WHAT this fragment carries and WHY,
+  #              as ids. Content-FREE by contract, so the context trace can report
+  #              which skills a turn injected without storing a byte of the bodies.
+  #              [] = nothing to name (the default for every provider that has no
+  #              natural id, e.g. the identity prompt).
+  #
+  #              The REASON is the point. A name alone answers "was something
+  #              injected"; the operator's actual question is "which skill did I
+  #              trigger, and why is it here" — `eager` (the agent always wants it),
+  #              `trigger:<matched phrase>` (this message asked for it), or absent
+  #              for a body a plugin supplied. String keys because these labels are
+  #              written to the context trace and to events as JSON: the round-trip
+  #              is then the identity, and no reader has to defend against both.
+  ContextFragment = Data.define(:content, :placement, :priority, :tokens,
+                                :source, :pinned, :labels) do
+    def self.build(content:, placement:, source:, priority: 50, tokens: nil,
+                   pinned: false, labels: [])
+      new(content: content, placement: placement, priority: priority,
+          tokens: tokens, source: source, pinned: pinned,
+          labels: Array(labels).map { |l| label(l) })
+    end
+
+    # A bare String is still a valid label (a provider that has an id but no reason
+    # to give) — it normalizes to a reason-less entry rather than being rejected.
+    def self.label(raw)
+      return { "name" => raw.to_s }.freeze unless raw.is_a?(Hash)
+
+      name = (raw[:name] || raw["name"]).to_s
+      reason = raw[:reason] || raw["reason"]
+      { "name" => name, "reason" => reason&.to_s }.compact.freeze
+    end
+  end
+end

data/lib/insika/context/priority.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Insika
+  module Context
+    # Precedence ladder for context fragments — SINGLE SOURCE of the order
+    # (trust boundary). Higher = more authority: appears earlier
+    # in the system prompt and survives budget cuts. Providers reference these
+    # constants instead of loose numbers, so the boundary is an auditable
+    # contract in a single place (trust_boundary_spec locks the order).
+    #
+    # Contract: identity and guardrails go PINNED at the top; TURN
+    # injections (request_context — the consumer's tenant/vars) sit at the
+    # BOTTOM and are sacrificed FIRST under budget. Identity (pinned) is NEVER
+    # truncated. A prompt injection riding in via turn data is DATA, not
+    # authority — it does not override IDENTITY/SOUL, and security decisions
+    # (tool allow/deny, egress, approvals) live in the engine/profile, never in
+    # the injected block.
+    module Priority
+      IDENTITY     = 100 # IDENTITY/SOUL (Prompt) — pinned
+      PROMPT_REF   = 90  # Prompt Catalog guardrails/refs (Prompt) — pinned
+      SKILL_BODY   = 85  # <active_skill> trigger-matched body (SkillTrigger)
+      SKILL        = 80  # <available_skills> level 1 (Skill)
+      MEMORY       = 75  # <memory> read path (Memory)
+      TOOL_SEARCH  = 70  # <available_tools> level 1 (ToolSearch)
+      HISTORY_MAX  = 79  # history ceiling by recency (Session)
+      HISTORY_BASE = 60  # history base; +idx up to the ceiling (Session)
+      REQUEST      = 40  # <request_context> — turn injection, the most cuttable
+    end
+  end
+end

data/lib/insika/context/provider.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Insika
+  # Base class for a context provider. Concrete
+  # subclasses live in Insika::Context::Providers. The base is
+  # deliberately minimal: `required?` lives on the provider, not the wiring.
+  class ContextProvider
+    def id = self.class.name       # override for a stable name
+    def required? = false          # true -> failure aborts the turn
+    def enabled_for?(_profile) = true
+    def call(_request) = []        # -> [ContextFragment]; may do IO
+  end
+
+  # Input for the provider contract.
+  #   session:    SessionStore::Session | nil
+  #   checkpoint: Checkpoint | nil (present on ResumeTask — history comes from it)
+  ContextRequest = Data.define(:session, :message, :profile, :tenant, :vars,
+                               :checkpoint)
+end

data/lib/insika/context/providers/memory.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Insika
+  module Context
+    module Providers
+      # Read path for cross-session memory. Thin adapter over
+      # the MemoryStore — same pattern as the Skill/ToolSearch provider: one
+      # `:system` fragment with the tenant's facts + recent notes. Deterministic
+      # (no embeddings/ranking).
+      class Memory < ContextProvider
+        def initialize(store:, notes_limit: 10)
+          @store = store
+          @notes_limit = notes_limit
+        end
+
+        # Per-agent opt-in. The Builder still applies the `context_providers`
+        # allowlist on top (two gates, like the other providers).
+        def enabled_for?(profile) = !!profile.memory
+
+        # required? == false (default): a failure (store unavailable) becomes a
+        # :provider_warning + graceful degradation — never aborts the turn.
+        def call(request)
+          tenant = memory_tenant(request)
+          facts = @store.facts(tenant: tenant)
+          notes = @store.notes(tenant: tenant, limit: @notes_limit)
+          return [] if facts.empty? && notes.empty?
+
+          # priority MEMORY (75): between skills (80) and deferred tools (70) in
+          # the sacrifice order. pinned false (cuttable under a tight budget).
+          [ContextFragment.build(content: format_block(facts, notes),
+                                 placement: :system, priority: Context::Priority::MEMORY, source: id)]
+        end
+
+        private
+
+        # Engine memory scope: an EXPLICIT tenant from the Command wins
+        # (multi-merchant override); otherwise the SESSION (=chat) — engine-owner
+        # memory is per-chat. No session (one-shot) and no tenant -> nil (MemoryStore
+        # applies _default). Symmetric to the write path (`state.tenant` in the Executor).
+        def memory_tenant(request)
+          explicit = request.respond_to?(:tenant) ? request.tenant : nil
+          return explicit if explicit
+
+          request.respond_to?(:session) ? request.session&.id : nil
+        end
+
+        # Passive <memory> (no instruction — the HOW of writing lives in the `remember` tool).
+        def format_block(facts, notes)
+          lines = facts.map { |f| %(  <fact key="#{f.key}">#{f.value}</fact>) }
+          lines += notes.map { |n| "  <note>#{n.text}</note>" }
+          <<~BLOCK.strip
+            <memory>
+            #{lines.join("\n")}
+            </memory>
+          BLOCK
+        end
+      end
+    end
+  end
+end

data/lib/insika/context/providers/prompt.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Insika
+  module Context
+    module Providers
+      # Absorbs SystemPrompt + SOUL.md. Identity is
+      # PINNED, priority 100 — never cut. required?: an agent without identity is
+      # a WRONG agent, not a degraded one. prompt_refs: priority 90 pinned
+      # fragments, from the PromptCatalog (catalog defaults to nil).
+      #
+      # PER-AGENT identity. `profile.prompt_files` (file names)
+      # wins over the wiring's `files:` — this fixes the limitation of a new
+      # agent inheriting Bia's prompt. The content comes from `agent_files`
+      # (AgentFileStore, lives in the Store), with a File.read fallback
+      # for on-disk paths (compat/seed). Without prompt_files -> uses the wiring's
+      # `files:` (deployment default; byte-for-byte parity).
+      class Prompt < ContextProvider
+        def initialize(base: "", files: [], catalog: nil, agent_files: nil, system_files: nil)
+          @base = base
+          @files = Array(files)
+          @catalog = catalog
+          @agent_files = agent_files
+          @system_files = system_files
+        end
+
+        def required? = true
+
+        def call(request)
+          fragments = []
+          identity = build_identity(request.profile)
+          unless identity.empty?
+            fragments << ContextFragment.build(content: identity, placement: :system,
+                                               priority: Context::Priority::IDENTITY, source: id, pinned: true)
+          end
+          fragments.concat(ref_fragments(request.profile))
+          fragments
+        end
+
+        private
+
+        # Migrates the SystemPrompt#build concatenation INTACT (without skills_block).
+        # A SINGLE fragment preserves the internal base->files order (sorting
+        # acts only BETWEEN fragments) and guarantees byte-for-byte parity.
+        #
+        # profile.prompt_files (names) wins over @files (wiring): an agent with
+        # its own identity does not inherit the deployment's. Each source resolves
+        # via AgentFileStore (per agent) OR File.read (on-disk path) — in that
+        # order. Without prompt_files, falls back to the wiring's @files.
+        def build_identity(profile)
+          parts = [@base]
+          parts.concat(system_parts) # global system files, for EVERY agent
+          # The agent's OWN inline identity (`base_prompt` — what the DSL's
+          # `instructions` and a pack's manifest set). It was stored, round-tripped
+          # and advertised on the A2A agent card while never reaching the model:
+          # every root wires `base: ""`, so an agent whose identity was inline had
+          # NO identity at all, and a chatty model answered plausibly enough to hide
+          # it. Additive to prompt_files, not exclusive: an agent may carry both.
+          parts << profile&.base_prompt.to_s
+          sources = Array(profile&.prompt_files)
+          if sources.empty?
+            @files.each { |f| parts << File.read(f, encoding: "UTF-8") if File.exist?(f) }
+          else
+            sources.each { |src| parts << read_source(profile&.id, src.to_s) }
+          end
+          parts.reject { |p| p.nil? || p.strip.empty? }.join("\n\n")
+        end
+
+        # GLOBAL system files: apply to every agent,
+        # injected BEFORE the individual identity. Empty/absent store -> [] ->
+        # a prompt byte-for-byte identical to before (the injection only exists if
+        # the operator authored something). Lexicographic order (SystemFileStore#list).
+        def system_parts
+          return [] unless @system_files
+
+          @system_files.list.map { |name| @system_files.read(name).to_s }
+        end
+
+        # Per-agent store first, disk second (compat/seed).
+        def read_source(agent_id, src)
+          stored = agent_id && @agent_files&.read(agent_id, src)
+          return stored if stored
+          return File.read(src, encoding: "UTF-8") if File.exist?(src)
+
+          ""
+        end
+
+        def ref_fragments(profile)
+          refs = Array(profile.prompt_refs)
+          return [] if refs.empty?
+
+          refs.map do |name|
+            entry = @catalog&.find(name.to_s)
+            unless entry
+              raise ContextError.new("prompt_ref '#{name}' not found in the Prompt Catalog",
+                                     provider: id)
+            end
+
+            ContextFragment.build(content: entry.body, placement: :system,
+                                  priority: Context::Priority::PROMPT_REF, source: id, pinned: true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/insika/context/providers/request.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Insika
+  module Context
+    module Providers
+      # :system fragment with turn metadata (tenant, relevant vars).
+      # Nothing if there is no metadata. priority 40. Not required?
+      # (metadata may degrade).
+      class Request < ContextProvider
+        def call(request)
+          lines = []
+          lines << "tenant: #{request.tenant}" if request.tenant
+          # "history" is transcript (consumed by the Session provider), not turn
+          # metadata — it does not leak into the system's request_context. Keys
+          # prefixed with "__" are INTERNAL slots (e.g. the per-chat model pin
+          # "__llm__") — reserved, never rendered to the model.
+          request.vars.to_h.each do |k, v|
+            next if k.to_s == "history" || k.to_s.start_with?("__")
+
+            lines << "#{k}: #{v}"
+          end
+          return [] if lines.empty?
+
+          [ContextFragment.build(
+            content: "<request_context>\n#{lines.join("\n")}\n</request_context>",
+            placement: :system, priority: Context::Priority::REQUEST, source: id
+          )]
+        end
+      end
+    end
+  end
+end