insika 0.0.1 → 0.1.0

data/lib/insika/mcp_store.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Insika
+  # MCP instances authored at runtime. One
+  # record per instance in the ConfigStore (scope "mcp"), keyed by the name slug
+  # (`tavily`, `github`, ...). Holds transport/command/url, the `enabled` flag and
+  # a Hash of `env` credentials (tokens/keys the instance injects into the server).
+  #
+  # The credentials (`env`) NEVER leave here in plaintext to the UI: the display
+  # reads (`get`/`all`) mask EACH value with the `__OCULTO__` sentinel. Only
+  # `get_raw`/`all_raw` (consumed by an MCP client, never by the screen) return
+  # the real values. On write, the sentinel coming back preserves the value; a new
+  # string replaces it; "" clears it (see Insika::SecretMasking, the same pattern
+  # as the LLM api_keys).
+  #
+  # Current scope: durable config CRUD (the instances UI). Running an MCP client
+  # against these instances is later runtime work — the store is the editable
+  # source from now on.
+  class McpStore
+    include Coercion
+
+    SCOPE = "mcp"
+
+    def initialize(config_store:)
+      @cs = config_store
+    end
+
+    # -> MASKED Hash (env with sentinel) | nil.
+    def get(name)
+      mask(raw(name))
+    end
+
+    # -> Hash with REAL env | nil. Internal use (MCP client), never the screen.
+    def get_raw(name)
+      raw(name)
+    end
+
+    # -> [String] slugs, lexicographic order.
+    def names = @cs.keys(SCOPE)
+
+    # -> [Hash] all MASKED (for the UI).
+    def all
+      names.filter_map { |n| get(n) }
+    end
+
+    # -> [Hash] all with REAL env (for an MCP client). Never goes to the screen.
+    def all_raw
+      names.filter_map { |n| raw(n) }
+    end
+
+    # Upsert with per-env-key secret reconciliation. `attrs`
+    # (string|symbol keys):
+    #   name (required), transport, command, url, description,
+    #   enabled (bool), env ({ "KEY" => value|sentinel|"" })
+    # -> MASKED Hash (the stored record).
+    def upsert(attrs)
+      h = symbolize(attrs)
+      name = presence(h[:name])
+      raise Insika::ValidationError, "name is required" if name.nil?
+
+      existing = raw(name)
+      record = {
+        "name" => name,
+        "transport" => presence(h[:transport]) || "stdio",
+        "command" => presence(h[:command]),
+        "url" => presence(h[:url]),
+        "description" => presence(h[:description]),
+        "enabled" => h.fetch(:enabled, true) ? true : false,
+        "env" => reconcile_env(h[:env], existing && existing["env"])
+      }
+      @cs.put(SCOPE, name, record)
+      mask(record)
+    end
+
+    # -> bool (did it exist?).
+    def delete(name) = @cs.delete(SCOPE, name.to_s)
+
+    private
+
+    def raw(name) = @cs.get(SCOPE, name.to_s)
+
+    # Each env value becomes the sentinel (or disappears if empty) — never leaks plaintext.
+    def mask(record)
+      return nil if record.nil?
+
+      env = (record["env"] || {}).each_with_object({}) do |(k, v), acc|
+        acc[k] = SecretMasking.mask(v)
+      end
+      record.merge("env" => env)
+    end
+
+    # Reconciles the env received from the form against the stored one, key by key: a
+    # key that came as the sentinel is preserved; a new string replaces it; "" (or missing
+    # from the submission) clears it. NEW keys are added; old keys absent from the form
+    # are removed (the form sends the complete set of keys).
+    def reconcile_env(incoming, existing)
+      inc = stringify_hash(incoming)
+      old = existing || {}
+      inc.each_with_object({}) do |(k, v), acc|
+        value = SecretMasking.reconcile(v, old[k])
+        acc[k] = value unless value.nil? || value.to_s.empty?
+      end
+    end
+
+    def stringify_hash(obj)
+      return {} unless obj.is_a?(Hash)
+
+      obj.each_with_object({}) { |(k, v), acc| acc[k.to_s] = v }
+    end
+
+    def symbolize(attrs)
+      (attrs || {}).each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
+    end
+  end
+end

data/lib/insika/mcp_tool_ingestor.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  # LIVE MCP ingestion (Phase 7, Stage E / spec §4 D8): discovers the tools of an
+  # MCP instance at RUNTIME (no hand-written manifest) and ingests them as
+  # data-tools. Given an McpStore instance + an INJECTABLE MCP client
+  # (duck-typed: `#list_tools -> [{name, description, inputSchema}]`), it builds a
+  # ToolManifest and REUSES the Stage B ingestion path (the :import_tools Command:
+  # batch upsert into the ToolStore + hot reload + per-tool report + partial-
+  # failure isolation R4). The ToolManifest MCP adapter (`inputSchema`) is reused
+  # — no schema parsing here.
+  #
+  # GENERIC (NF1): nothing here mentions achei/openclaw. The MCP instance is DATA in the store.
+  #
+  # BINDING STRATEGY (this stage's choice, bounded):
+  #   Each discovered tool becomes an HTTP data-tool that makes a JSON-RPC 2.0
+  #   `tools/call` POST to the instance endpoint (url). The tool name is resolved
+  #   at INGESTION (literal in the body); the model's arguments come in as `{{param}}`
+  #   per TOP-level property of the inputSchema (with quoting by type — strings
+  #   quoted, others raw, via the DataDefinedTool :body encode). This way the tool
+  #   runs through the SAME HTTP path as the other data-tools (egress guard, secret
+  #   headers, hot reload) — no new execution code.
+  #
+  #   Each tool gets `group: "mcp:<instance>"` so the Stage C per-group gating
+  #   (tools_allow_groups) works for free.
+  #
+  # DEFERRED / OUT-OF-SCOPE (documented — spec §4 D8):
+  #   - Real MCP transport: only instances with a `url` (http transport) are ingestible;
+  #     stdio has no HTTP endpoint -> raises a clear error (later work).
+  #   - MCP session lifecycle (initialize/negotiation/session-id/notifications) and the
+  #     UNWRAP of the `tools/call` response (`{content:[{type,text}]}`) — the binding
+  #     makes a stateless POST and returns the raw body (extract body_raw).
+  #   - Credential injection (the instance `env`) as an auth header in the HTTP
+  #     binding: the `env` is consumed by a real MCP client (deferred), not mapped
+  #     to a header here.
+  #   - Tools whose name/top-level property is outside the ToolDefinition NAME_RE
+  #     (uppercase/hyphens) are ISOLATED into `errors[]` by the import (R4).
+  class McpToolIngestor
+    def initialize(mcp_store:, import_tools:, client_factory: nil)
+      @mcp_store = mcp_store
+      @import_tools = import_tools
+      # Per-instance client factory (default: minimal JSON-RPC HTTP client).
+      # Injectable for tests (Fake) and to swap for a real transport later.
+      @client_factory = client_factory || method(:default_client)
+    end
+
+    # Discovers + ingests the tools of instance `name`. `client` is injectable
+    # (Fake in tests); absent -> the factory builds one from the record. -> the
+    # import_tools report + `instance:` ({ instance:, version:, created:, updated:, errors: }).
+    def ingest(name, client: nil)
+      manifest = manifest_for(name, client: client)
+      report = @import_tools.call(Insika::Command.build(:import_tools, manifest, transport: :internal))
+      report.merge(instance: name.to_s)
+    end
+
+    # Discovers the tools and builds the manifest Hash (without ingesting) — isolable for testing.
+    def manifest_for(name, client: nil)
+      record = @mcp_store.get_raw(name.to_s)
+      raise Insika::NotFoundError, "MCP instance '#{name}' not found" if record.nil?
+      raise Insika::ValidationError, "MCP instance '#{name}' is disabled" unless record["enabled"]
+
+      url = presence(record["url"])
+      if url.nil?
+        raise Insika::ValidationError,
+              "MCP instance '#{name}' has no url: live ingestion requires HTTP transport " \
+              "(stdio is later work — D8)"
+      end
+
+      tools = Array((client || @client_factory.call(record)).list_tools)
+      build_manifest(name.to_s, url, tools)
+    end
+
+    private
+
+    def build_manifest(name, url, tools)
+      {
+        "version" => 1,
+        "defaults" => {
+          "method" => "POST",
+          "headers" => { "Content-Type" => "application/json" },
+          "response" => { "extract" => "body_raw" },
+          "group" => "mcp:#{name}"
+        },
+        "tools" => tools.map { |raw| tool_entry(name, url, stringify(raw)) }
+      }
+    end
+
+    # Raw MCP entry -> manifest entry (MCP `inputSchema` envelope + JSON-RPC
+    # binding). ToolManifest normalizes the `inputSchema` (MCP adapter) and
+    # inherits the defaults; `group` falls through from the defaults.
+    def tool_entry(name, url, raw)
+      tool_name = raw["name"]
+      input_schema = raw["inputSchema"] || {}
+      {
+        "name" => tool_name,
+        "description" => presence(raw["description"]) || "Tool '#{tool_name}' from MCP server '#{name}'.",
+        "inputSchema" => input_schema,
+        "url" => url,
+        "side_effect" => true, # a tools/call is a side effect (checkpoint/skip-on-resume)
+        "body" => jsonrpc_call_body(tool_name, input_schema)
+      }
+    end
+
+    # JSON-RPC 2.0 `tools/call` body. `name` literal (resolved at ingestion);
+    # `arguments` per TOP-level property of the inputSchema, with `{{param}}` that
+    # the DataDefinedTool interpolates at TURN time.
+    def jsonrpc_call_body(tool_name, input_schema)
+      %({"jsonrpc":"2.0","id":1,"method":"tools/call",) +
+        %("params":{"name":#{JSON.generate(tool_name)},"arguments":#{arguments_fragment(input_schema)}}})
+    end
+
+    # -> "{...}" JSON with one placeholder per top-level property. Strings are
+    # quoted (the DataDefinedTool :body encode returns the escaped content WITHOUT
+    # quotes); other types raw (the encode returns value.to_json).
+    def arguments_fragment(input_schema)
+      props = (input_schema["properties"] || input_schema[:properties] || {})
+      return "{}" if props.nil? || props.empty?
+
+      pairs = props.map do |key, spec|
+        type = stringify(spec)["type"].to_s
+        placeholder = type == "string" ? %("{{#{key}}}") : "{{#{key}}}"
+        %(#{JSON.generate(key.to_s)}:#{placeholder})
+      end
+      "{#{pairs.join(',')}}"
+    end
+
+    def default_client(record)
+      Insika::McpHttpClient.new(url: record["url"])
+    end
+
+    def presence(str) = Insika::Coercion.presence(str)
+
+    def stringify(obj)
+      case obj
+      when Hash then obj.each_with_object({}) { |(k, v), acc| acc[k.to_s] = stringify(v) }
+      when Array then obj.map { |v| stringify(v) }
+      else obj
+      end
+    end
+  end
+end

data/lib/insika/memory_store.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "time"
+
+module Insika
+  # AGENT-MEMORY DOMAIN store. Two layers
+  # scoped per tenant over any `Insika::Store`: `profile` (stable key-value
+  # facts) and `notes` (append-only free-form notes). Mirrors the
+  # `PendingActionStore` (normalizes symbol→string on write, O(n) scan on
+  # read, records with a timestamp).
+  #
+  # NOT to be confused with `Insika::Stores::Memory` (in-memory KV backend):
+  # this is the domain store; that one is one of the backends this writes to.
+  class MemoryStore
+    SCOPE_PREFIX = "memory"       # scope = "memory:<tenant>"
+    FACT_PREFIX  = "fact:"
+    NOTE_PREFIX  = "note:"
+    DEFAULT_TENANT = "_default"   # no tenant in the Command
+
+    Fact = Data.define(:key, :value, :updated_at)
+    Note = Data.define(:id, :text, :created_at)
+
+    def initialize(store:)
+      @store = store
+    end
+
+    # Upsert (last-write-wins, Store contract). -> Fact
+    def put_fact(tenant:, key:, value:)
+      record = { "key" => key.to_s, "value" => stringify(value), "updated_at" => timestamp }
+      @store.set(scope_for(tenant), FACT_PREFIX + key.to_s, record)
+      to_fact(record)
+    end
+
+    # -> Fact | nil
+    def get_fact(tenant:, key:)
+      record = @store.get(scope_for(tenant), FACT_PREFIX + key.to_s)
+      record && to_fact(record)
+    end
+
+    # -> [Fact] sorted by key (list is lexicographic).
+    def facts(tenant:)
+      scope = scope_for(tenant)
+      @store.list(scope, FACT_PREFIX).filter_map do |k|
+        record = @store.get(scope, k)
+        record && to_fact(record)
+      end
+    end
+
+    # -> bool (did it exist?)
+    def forget_fact(tenant:, key:)
+      @store.delete(scope_for(tenant), FACT_PREFIX + key.to_s)
+    end
+
+    # Append. `at` (ISO8601) goes at the START of the key so `list` returns the notes in
+    # chronological order; `id`/`at` injectable for deterministic tests. -> Note
+    def add_note(tenant:, text:, id: SecureRandom.uuid, at: nil)
+      at ||= timestamp
+      record = { "id" => id.to_s, "text" => text.to_s, "created_at" => at }
+      @store.set(scope_for(tenant), NOTE_PREFIX + "#{at}:#{id}", record)
+      to_note(record)
+    end
+
+    # -> [Note] MOST RECENT first, capped by `limit`.
+    def notes(tenant:, limit: nil)
+      scope = scope_for(tenant)
+      keys = @store.list(scope, NOTE_PREFIX).reverse # list is chronological -> reverse = most recent first
+      keys = keys.first(limit) if limit
+      keys.filter_map do |k|
+        record = @store.get(scope, k)
+        record && to_note(record)
+      end
+    end
+
+    private
+
+    def scope_for(tenant) = "#{SCOPE_PREFIX}:#{tenant.nil? || tenant.to_s.empty? ? DEFAULT_TENANT : tenant}"
+
+    def to_fact(record) = Fact.new(key: record["key"], value: record["value"], updated_at: record["updated_at"])
+    def to_note(record) = Note.new(id: record["id"], text: record["text"], created_at: record["created_at"])
+
+    def timestamp = Time.now.utc.iso8601
+
+    def stringify(obj)
+      case obj
+      when Hash then obj.each_with_object({}) { |(k, v), acc| acc[k.to_s] = stringify(v) }
+      when Array then obj.map { |v| stringify(v) }
+      when Symbol then obj.to_s
+      else obj
+      end
+    end
+  end
+end

data/lib/insika/message_origin.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Insika
+  # WHO PRODUCED A TRANSCRIPT MESSAGE — the field a `role` cannot carry.
+  #
+  # `role` says where a message sits in the conversation, not who wrote it, and the
+  # two come apart constantly:
+  #
+  # · The engine delivers an async subagent's result to the parent as a NEW turn
+  #   (RFC-0010) — a `user` message the engine wrote.
+  # · A guardrail short-circuits with a safe reply — an `assistant` message produced
+  #   with zero LLM calls.
+  # · A consumer composes context blocks into the input it sends (`<memoria> …`,
+  #   `<cacau_cep_obrigatorio> …`) — a `user` message the customer never typed.
+  # · In an imported transcript, a human operator types after a handoff — an
+  #   `assistant` message no model produced.
+  #
+  # Reading a transcript without that distinction is not a rounding error. The first
+  # refinement run over real traffic reported `repetition ×219` on one agent: every
+  # one of them the engine reading its own injected fragment back and calling it a
+  # customer repeating themselves (PR #133). That was filtered by a REGEX on the
+  # leading tag, labelled in the code as a heuristic standing in for this field.
+  #
+  # ABSENT is the common case and stays valid forever: a message with no origin is
+  # read as the natural producer for its role (`user` → the customer, `assistant` →
+  # the agent). Nothing in the existing corpus, the stores or the pilot's database
+  # has to be migrated, and a reader that ignores the field is exactly as correct as
+  # it was before.
+  module MessageOrigin
+    KEY = "origin"
+
+    CUSTOMER = "customer" # a person on the user side (the default for `user`)
+    AGENT    = "agent"    # the model (the default for `assistant`)
+    ENGINE   = "engine"   # Insika itself, or the consumer composing on its behalf
+    OPERATOR = "operator" # a HUMAN on the assistant side (a handoff; set by importers)
+
+    ALL = [CUSTOMER, AGENT, ENGINE, OPERATOR].freeze
+
+    module_function
+
+    # A declared origin, or nil. Anything outside the closed set is REFUSED rather
+    # than stored: an unknown value would silently read as "absent" downstream, and
+    # a typo'd marker is worse than none — it looks like the filtering is on.
+    def parse!(value)
+      return nil if Coercion.blank?(value)
+
+      name = value.to_s.strip.downcase
+      return name if ALL.include?(name)
+
+      raise Insika::ValidationError,
+            "unknown message origin: #{value.inspect} (expected #{ALL.join(', ')})"
+    end
+
+    # Did a PERSON on the user side write this? Absent origin = yes, because that is
+    # what a `user` message meant before this field existed.
+    def customer?(message)
+      message["role"].to_s == "user" && [nil, CUSTOMER].include?(origin_of(message))
+    end
+
+    # Did the MODEL write this? Absent origin = yes, same reasoning.
+    def agent?(message)
+      message["role"].to_s == "assistant" && [nil, AGENT].include?(origin_of(message))
+    end
+
+    def origin_of(message)
+      v = message[KEY] || message[KEY.to_sym]
+      Coercion.presence(v)&.downcase
+    end
+
+    # Stamps a message hash, leaving it untouched when there is nothing to declare —
+    # so the common turn keeps producing exactly the two-key shape it always did.
+    def stamp(message, origin)
+      origin.nil? ? message : message.merge(KEY => origin)
+    end
+  end
+end

data/lib/insika/middleware.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Insika
+  # A link in the chain (stage 4). A Middleware MODIFIES the TurnState,
+  # short-circuits, and has operational effect (rate limit, tracing, cost) — it
+  # does NOT decide tool/skill permission (that is Policy). Short-circuit =
+  # NOT calling `nxt` and setting `state.halt_reason`. Setting halt_reason AND
+  # calling nxt is a contract violation (the Executor prioritizes halt_reason on
+  # the way back).
+  #
+  # Concurrency: it runs on the task's fiber; IO (e.g. a tracing exporter)
+  # must be async off the path (`Async { ... }` fire-and-forget) or accept
+  # the latency in the turn. No timeout of its own (covered by the turn timeout).
+  class Middleware
+    def call(state, &nxt)
+      nxt.call(state) # default link: pass-through
+    end
+  end
+
+  # Rack-like composition: registration order = execution order (the first is
+  # the outermost link). It does NOT rescue (an exception propagates as a turn
+  # failure) nor does it have a special halt mechanism — the short-circuit is
+  # structural (the link does not call nxt).
+  class MiddlewareStack
+    def initialize(middlewares = [])
+      @middlewares = middlewares
+    end
+
+    def call(state, &terminal)
+      chain = @middlewares.reverse.reduce(terminal) do |nxt, mw|
+        proc { |s| mw.call(s, &nxt) }
+      end
+      chain.call(state)
+    end
+  end
+end

data/lib/insika/model_policy.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Insika
+  # Governance of WHICH models an agent may use (`profile.model_policy`).
+  # Cheap allowlist: exact refs + wildcard-by-provider — evaluated against the
+  # RESOLVED (model, provider) at turn start (ModelResolver), so a chat override
+  # can never pin a model outside the agent's fence.
+  #
+  # A ref is a String in one of three shapes:
+  #   "provider/model"  -> matches that exact provider AND model
+  #   "provider/*"      -> matches ANY model of that provider (wildcard)
+  #   "model"           -> matches that model under ANY provider (provider-agnostic)
+  #
+  # `allow` semantics follow the ONE-allowlist rule of the project:
+  #   nil = NO policy (every model allowed — parity, the common case)
+  #   []  = NOTHING allowed (deny-all; an agent so configured cannot resolve a model)
+  #   [refs] = the model must match at least one ref
+  module ModelPolicy
+    module_function
+
+    # policy: `profile.model_policy` — a Hash `{ "allow" => [refs] }` | nil.
+    # nil policy OR a nil/absent `allow` list = allowed (no fence). -> bool.
+    def allowed?(policy, model:, provider:)
+      allow = allow_list(policy)
+      return true if allow.nil? # no fence
+
+      model = model.to_s
+      provider = provider.to_s
+      allow.any? { |ref| match?(ref.to_s, model, provider) }
+    end
+
+    # Extracts the allow list (string|symbol key), nil when there is no fence.
+    def allow_list(policy)
+      return nil if policy.nil?
+
+      list = policy["allow"] || policy[:allow]
+      list.nil? ? nil : Array(list)
+    end
+
+    def match?(ref, model, provider)
+      if ref.include?("/")
+        ref_provider, ref_model = ref.split("/", 2)
+        return false unless ref_provider == provider
+
+        ref_model == "*" || ref_model == model
+      else
+        # provider-agnostic: match the model under any provider.
+        ref == model
+      end
+    end
+  end
+end