insika 0.0.1 → 0.2.0

data/lib/insika/model_policy.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Insika
+  # Governance of WHICH models an agent may use (`profile.model_policy`).
+  # Cheap allowlist: exact refs + wildcard-by-provider — evaluated against the
+  # RESOLVED (model, provider) at turn start (ModelResolver), so a chat override
+  # can never pin a model outside the agent's fence.
+  #
+  # A ref is a String in one of three shapes:
+  #   "provider/model"  -> matches that exact provider AND model
+  #   "provider/*"      -> matches ANY model of that provider (wildcard)
+  #   "model"           -> matches that model under ANY provider (provider-agnostic)
+  #
+  # `allow` semantics follow the ONE-allowlist rule of the project:
+  #   nil = NO policy (every model allowed — parity, the common case)
+  #   []  = NOTHING allowed (deny-all; an agent so configured cannot resolve a model)
+  #   [refs] = the model must match at least one ref
+  module ModelPolicy
+    module_function
+
+    # policy: `profile.model_policy` — a Hash `{ "allow" => [refs] }` | nil.
+    # nil policy OR a nil/absent `allow` list = allowed (no fence). -> bool.
+    def allowed?(policy, model:, provider:)
+      allow = allow_list(policy)
+      return true if allow.nil? # no fence
+
+      model = model.to_s
+      provider = provider.to_s
+      allow.any? { |ref| match?(ref.to_s, model, provider) }
+    end
+
+    # Extracts the allow list (string|symbol key), nil when there is no fence.
+    def allow_list(policy)
+      return nil if policy.nil?
+
+      list = policy["allow"] || policy[:allow]
+      list.nil? ? nil : Array(list)
+    end
+
+    def match?(ref, model, provider)
+      if ref.include?("/")
+        ref_provider, ref_model = ref.split("/", 2)
+        return false unless ref_provider == provider
+
+        ref_model == "*" || ref_model == model
+      else
+        # provider-agnostic: match the model under any provider.
+        ref == model
+      end
+    end
+  end
+end

data/lib/insika/model_resolver.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module Insika
+  # LLM config v2 resolution. Turns the three config layers into a
+  # single ModelSelection at turn start:
+  #
+  #   Chat (per-session pin) > Agent (profile.model) > Platform default (Settings)
+  #
+  # Also enforces the agent's `model_policy` on the RESOLVED model (a chat pin can
+  # never escape the fence) and resolves the fallback chain + selection source.
+  #
+  # Pure: no ruby_llm, no store writes. `settings_store` is optional — nil means
+  # "no platform layer" (an agent WITHOUT a model then fails high), preserving the
+  # pre-v2 behavior for the many call sites that build an Executor without one.
+  class ModelResolver
+    # Reserved, collision-safe slot for the per-session model pin inside
+    # `session.vars` (namespaced so it never renders in <request_context> — the
+    # Request provider skips "__"-prefixed vars).
+    SESSION_SLOT = "__llm__"
+
+    def initialize(settings_store: nil)
+      @settings_store = settings_store
+    end
+
+    # The layer that won resolution, before policy/fallback processing.
+    # `source` is where the model came from (:chat/:agent/:platform_default).
+    Choice = Data.define(:model, :provider, :source, :pinned)
+    private_constant :Choice
+
+    # profile: AgentProfile; session: SessionStore::Session | nil.
+    # -> ModelSelection. Raises PolicyDenied (model outside the agent's fence) or
+    # Error (no model resolvable at any layer).
+    def resolve(profile:, session: nil)
+      settings = platform_settings
+      choice = pick(profile, session, settings)
+
+      if blank?(choice.model)
+        raise Insika::Error,
+              "no model resolved for agent '#{profile.id}': set the agent model or a platform default_model (Settings)"
+      end
+
+      provider = choice.provider&.to_sym
+      enforce_policy!(profile, choice.model, provider)
+
+      ModelSelection.new(
+        model: choice.model, provider: provider, source: choice.source, pinned: choice.pinned,
+        params: resolve_params(profile, session, settings, choice.model, provider),
+        fallbacks: choice.pinned ? [] : resolve_fallbacks(profile, settings, choice.model, provider)
+      )
+    end
+
+    private
+
+    # Chat pin > agent model > platform default.
+    def pick(profile, session, settings)
+      pin = session_pin(session)
+      if pin && !blank?(pin[:model])
+        Choice.new(model: pin[:model], provider: pin[:provider], source: :chat, pinned: true)
+      elsif !blank?(profile.model)
+        Choice.new(model: profile.model, provider: profile.provider, source: :agent, pinned: false)
+      else
+        Choice.new(model: settings["default_model"], provider: settings["default_provider"],
+                   source: :platform_default, pinned: false)
+      end
+    end
+
+    # Per-session pin from vars["__llm__"] = { "model" =>, "provider" => }.
+    # SessionStore deep_stringifies vars on write, so the slot is always
+    # string-keyed. -> { model:, provider: } | nil.
+    def session_pin(session)
+      slot = session&.vars&.dig(SESSION_SLOT)
+      return nil unless slot.is_a?(Hash)
+
+      { model: slot["model"], provider: slot["provider"] }
+    end
+
+    def enforce_policy!(profile, model, provider)
+      policy = model_policy(profile)
+      return if ModelPolicy.allowed?(policy, model: model, provider: provider)
+
+      raise Insika::PolicyDenied.new(
+        policy: :model_policy,
+        reason: "model '#{ref(model, provider)}' not allowed for agent '#{profile.id}' " \
+                "(allow: #{ModelPolicy.allow_list(policy).inspect})"
+      )
+    end
+
+    # Platform fallback chain (Settings["fallback_models"]) minus the primary,
+    # filtered by the agent's model_policy. Each entry: "provider/model" | "model".
+    def resolve_fallbacks(profile, settings, primary_model, primary_provider)
+      policy = model_policy(profile)
+      Array(settings["fallback_models"]).filter_map do |entry|
+        model, provider = parse_ref(entry)
+        next if model.nil?
+        next if model == primary_model && provider == primary_provider # dedupe the primary
+        next unless ModelPolicy.allowed?(policy, model: model, provider: provider)
+
+        { model: model, provider: provider }
+      end
+    end
+
+    # "provider/model" -> [model, :provider]; "model" -> [model, nil].
+    def parse_ref(entry)
+      s = entry.to_s.strip
+      return [nil, nil] if s.empty?
+
+      if s.include?("/")
+        provider, model = s.split("/", 2)
+        [model, presence(provider)&.to_sym]
+      else
+        [s, nil]
+      end
+    end
+
+    def platform_settings
+      @settings_store ? @settings_store.get : {}
+    end
+
+    # AgentProfile always carries `params` (a string-keyed Hash, {} when unset)
+    # and `model_policy` (nil = no fence) — no feature-detection needed.
+    def profile_params(profile) = profile.params
+    def model_policy(profile) = profile.model_policy
+
+    # Generation params: the agent's temperature/max_tokens plus the reasoning
+    # control (`thinking`) resolved across the 4 layers. A resolved nil means
+    # "nothing set anywhere" -> drop the key so the provider default stands.
+    def resolve_params(profile, session, settings, model, provider)
+      params = normalize_params(profile_params(profile))
+      thinking = resolve_thinking(profile, session, settings, model, provider)
+      thinking.nil? ? params.tap { |p| p.delete(:thinking) } : params.merge(thinking: thinking)
+    end
+
+    # First non-blank of Chat > Agent > Model-default > Global (most specific wins).
+    def resolve_thinking(profile, session, settings, model, provider)
+      [
+        chat_thinking(session),
+        presence(profile.params["thinking"]),
+        model_thinking(settings, model, provider),
+        presence(settings["thinking"])
+      ].find { |v| !blank?(v) }
+    end
+
+    # Per-chat override: the `thinking` slot in the session pin (vars["__llm__"]),
+    # independent of whether the chat also pinned a model.
+    def chat_thinking(session)
+      slot = session&.vars&.dig(SESSION_SLOT)
+      slot.is_a?(Hash) ? presence(slot["thinking"]) : nil
+    end
+
+    # Per-model default: settings["model_params"][ref]["thinking"], matched by the
+    # full "provider/model" ref first, then the bare model id.
+    def model_thinking(settings, model, provider)
+      map = settings["model_params"]
+      return nil unless map.is_a?(Hash)
+
+      ref = provider ? "#{provider}/#{model}" : nil
+      entry = (ref && map[ref]) || map[model.to_s]
+      entry.is_a?(Hash) ? presence(entry["thinking"]) : nil
+    end
+
+    # Coerces authored params (string|symbol keys from the JSON round-trip) into
+    # the symbol shape ModelSelection#apply_params consumes.
+    def normalize_params(params)
+      out = {}
+      params.each do |k, v|
+        key = k.to_sym
+        out[key] = v if %i[temperature max_tokens thinking].include?(key)
+      end
+      out
+    end
+
+    def ref(model, provider) = provider ? "#{provider}/#{model}" : model.to_s
+    def blank?(v) = Insika::Coercion.blank?(v)
+    def presence(v) = Insika::Coercion.presence(v)
+  end
+end

data/lib/insika/model_selection.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Insika
+  # The RESOLVED model decision for a turn (ModelResolver output). Immutable
+  # snapshot carried on the TurnState, applied to the RubyLLM chat at stage 5 and
+  # surfaced in the turn's usage for telemetry/billing.
+  #
+  # Fields:
+  #   model     -> the resolved model id (String) | nil (RubyLLM's own default)
+  #   provider  -> Symbol | nil (nil => RubyLLM infers from the model registry)
+  #   source    -> :chat | :agent | :platform_default (WHERE the model came from)
+  #   pinned    -> true when the model is a USER pin (per-chat override). A pin
+  #                "fails HIGH": no silent fallback (OpenClaw semantics). An agent
+  #                model / platform default is NOT pinned -> fallbacks apply.
+  #   params    -> generation params (Hash of symbols: temperature/max_tokens/thinking)
+  #   fallbacks -> ordered [{ model:, provider: }] to try when NOT pinned. The
+  #                mid-turn ROTATION across this chain (plus the profile's own
+  #                reliability["fallback"] refs) is WS3's Reliability coordinator;
+  #                here the chain is resolved + surfaced (source/pinned) for it.
+  ModelSelection = Data.define(:model, :provider, :source, :pinned, :params, :fallbacks) do
+    def initialize(model:, provider: nil, source: :platform_default, pinned: false,
+                   params: {}, fallbacks: [])
+      super
+    end
+
+    def pinned? = pinned
+
+    # provider present => tell RubyLLM to trust the id (skip the registry lookup),
+    # matching the pre-v2 create_chat behavior.
+    def assume_model_exists? = !provider.nil?
+
+    # Applies the generation params to a RubyLLM chat (or any object exposing the
+    # same `with_*` fluent API). Pure and duck-typed: a `with_*` the target does
+    # not expose is skipped (a fake chat in specs need only implement the ones it
+    # asserts). RubyLLM raises on a nil arg, so only PRESENT params are applied.
+    # Returns the chat (the `with_*` calls mutate and return self).
+    def apply_params(chat)
+      p = params || {}
+      apply(chat, :with_temperature, p[:temperature]) if numeric?(p[:temperature])
+      apply_payload_params(chat, p)
+      apply_effort(chat, p[:thinking]) if present?(p[:thinking])
+      chat
+    end
+
+    private
+
+    def apply(chat, method, value)
+      chat.public_send(method, value) if chat.respond_to?(method)
+    end
+
+    # The params that ride RubyLLM's `with_params` (deep-merged OVER the provider
+    # payload, so they win over a provider default). ONE call on purpose:
+    # `with_params` REPLACES the whole hash — a second call would silently drop
+    # the first one's keys.
+    #
+    # max_tokens goes here because the gem has NO with_max_* setter: the chat's
+    # fluent API covers temperature/thinking only, and Anthropic is the only
+    # provider that emits a cap at all (from the model registry, 4096 by default).
+    # `max_tokens` is the wire key for Anthropic and for every OpenAI-compatible
+    # provider (DeepSeek included); OpenAI's o-series wants
+    # `max_completion_tokens` instead — that per-provider mapping is not modelled
+    # yet, and a raw provider key is not authorable (the profile whitelists
+    # temperature/max_tokens/thinking).
+    def apply_payload_params(chat, p)
+      return unless chat.respond_to?(:with_params)
+
+      payload = {}
+      payload[:max_tokens] = p[:max_tokens] if numeric?(p[:max_tokens])
+      toggle = thinking_toggle(p[:thinking])
+      payload[:thinking] = { type: toggle } if toggle
+      chat.with_params(**payload) unless payload.empty?
+    end
+
+    # The resolved reasoning control (4-layer). Two axes folded into one field:
+    #   off              -> reasoning DISABLED (thinking:{type:disabled})
+    #   on               -> reasoning ENABLED, provider-default effort
+    #   low|medium|high  -> reasoning enabled at that effort (reasoning_effort)
+    # Blank/nil never reaches here (apply_params guards on present?), so no config
+    # anywhere = the provider's own default.
+    #
+    # ruby_llm 1.16's with_thinking only EMITS reasoning_effort and cannot toggle
+    # on/off; the on/off ride on with_params (deep-merged over the payload). The
+    # thinking:{type:} shape is DeepSeek's OpenAI-compat contract — gated to it
+    # (nil provider = the platform default, DeepSeek here); other providers get
+    # effort-only until their toggle wire is mapped (e.g. Anthropic output_config).
+    def thinking_toggle(value)
+      return nil unless present?(value) && reasoning_toggle_supported?
+
+      case value.to_s
+      when "off" then "disabled"
+      when "on" then "enabled"
+      end
+    end
+
+    def apply_effort(chat, value)
+      return unless %w[low medium high].include?(value.to_s)
+
+      chat.with_thinking(effort: value.to_sym) if chat.respond_to?(:with_thinking)
+    end
+
+    def reasoning_toggle_supported?
+      provider.nil? || provider.to_sym == :deepseek
+    end
+
+    def numeric?(v) = v.is_a?(Numeric)
+    def present?(v) = Insika::Coercion.present?(v)
+  end
+
+  # The selectable reasoning values (4-layer). Blank/absent = inherit the
+  # broader layer; these are the explicit choices. Shared with the Studio forms.
+  # Defined on the class OUTSIDE the Data.define block on purpose: a constant
+  # assigned inside the block would land in the enclosing lexical scope (Insika),
+  # not on ModelSelection.
+  ModelSelection::THINKING_LEVELS = %w[off on low medium high].freeze
+end

data/lib/insika/onboarding.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+module Insika
+  # LLM-first onboarding surface. The "Flue trick": the insika
+  # serves, from itself, a `start.md` addressed to the DEVELOPER'S OWN coding agent
+  # ("Read <base>/start.md then help me build my first agent") plus a machine-readable
+  # `/models.json` and the public docs mirrored as raw markdown. It is `rails new`
+  # reimplemented as a prompt, with the generator being the coding agent the developer
+  # already has.
+  #
+  # Pure and data-defined: everything it serves comes from injected sources — a
+  # `start.md` TEMPLATE file, a NAME=>path map of public docs, and (optionally) the
+  # SettingsStore / LLMProviderStore / the served agents. It only READS (no writes, no
+  # RubyLLM, no Executor), so the transport can call it under the constitutional rule.
+  # Secrets never leak: `models_json` reads the MASKED provider view and omits base
+  # urls/keys entirely — only slugs and model ids, which is all a coding agent needs to
+  # write a correct `model`/`provider` line.
+  #
+  # Injection (all optional but `template_path`/`docs`):
+  #   template_path  -> the start.md source (a real file, so it stays reviewable/raw)
+  #   docs           -> { "slug" => "/abs/path.md", … } served at GET /docs/:slug.md.
+  #                     An explicit ALLOWLIST — the gitignored internal docs are never
+  #                     in it, and there is no filesystem traversal (only map keys).
+  #   settings_store -> platform default_model / fallbacks / thinking (nil = omit)
+  #   provider_store -> configured providers + their model ids, MASKED (nil = omit)
+  #   agents         -> callable returning [{ id:, model:, provider:, description: }]
+  #                     for the agents already served here (nil = none)
+  class Onboarding
+    # Bumped when the models.json SHAPE changes (its own contract, independent of the
+    # settings schema). A consuming coding agent can branch on it.
+    MODELS_SCHEMA_VERSION = 1
+
+    # The PUBLIC docs allowlist, repo-relative: slug => path. Explicit on purpose —
+    # the gitignored internal docs (/ TRANSLATION-TRACKER / …)
+    # are NEVER here, so /docs can only ever serve OSS material. Kept in sync with the
+    # tracked `.md` prose (README + docs/*.md).
+    PUBLIC_DOCS = {
+      "readme" => "README.md",
+      "why" => "docs/WHY.md",
+      "agents" => "docs/AGENTS.md",
+      "tools" => "docs/TOOLS.md",
+      "skills" => "docs/SKILLS.md",
+      "context" => "docs/CONTEXT.md",
+      "workflows" => "docs/WORKFLOWS.md",
+      "channels" => "docs/CHANNELS.md",
+      "plugins" => "docs/PLUGINS.md",
+      "security" => "docs/SECURITY.md",
+      "architecture" => "docs/ARCHITECTURE.md",
+      "running-local" => "docs/RUNNING-LOCAL.md",
+      "deploy" => "docs/DEPLOY.md",
+      "embedding" => "docs/EMBEDDING.md",
+      "sandbox" => "docs/SANDBOX.md",
+      "benchmark" => "docs/BENCHMARK.md",
+      "observability" => "docs/OBSERVABILITY.md",
+      "evals" => "docs/EVALS.md",
+      "refinement" => "docs/REFINEMENT.md",
+      "loadtest" => "docs/LOADTEST.md",
+      "releasing" => "docs/RELEASING.md"
+    }.freeze
+
+    # Repo-relative path to the start.md template.
+    TEMPLATE = "docs/onboarding/start.md"
+
+    # Builds the standard onboarding surface rooted at `root` (the repo/gem root),
+    # wiring the PUBLIC_DOCS allowlist + start.md template. The three composition roots
+    # (minimal wiring, DSL serve, deployment) pass their own stores/agents on top. A
+    # doc whose file is absent (a slimmed-down gem) is simply dropped — never a boot
+    # failure.
+    def self.standard(root:, settings_store: nil, provider_store: nil, agents: nil)
+      docs = PUBLIC_DOCS.each_with_object({}) do |(slug, rel), acc|
+        path = File.join(root, rel)
+        acc[slug] = path if File.file?(path)
+      end
+      new(template_path: File.join(root, TEMPLATE), docs: docs,
+          settings_store: settings_store, provider_store: provider_store, agents: agents)
+    end
+
+    def initialize(template_path:, docs: {}, settings_store: nil, provider_store: nil, agents: nil)
+      @template_path = template_path
+      @docs = docs || {}
+      @settings_store = settings_store
+      @provider_store = provider_store
+      @agents = agents # callable -> [Hash] | nil
+    end
+
+    # The onboarding skill (raw markdown), with the live base url interpolated so the
+    # coding agent knows where to fetch the models list and docs. Read on each request
+    # (the file is small and this surface is low-traffic) — editing start.md needs no
+    # restart.
+    def start_md(base_url:)
+      base = normalize_base(base_url)
+      substitute(File.read(@template_path), base)
+    end
+
+    # Machine-readable model catalog. Everything a coding agent needs to write a valid
+    # `model`/`provider` line, and nothing secret. Sources that are absent (nil store)
+    # simply drop their key — a fresh DSL serve still returns a coherent document
+    # (served agents + thinking levels + whatever default is set).
+    def models_json(base_url:)
+      base = normalize_base(base_url)
+      settings = @settings_store&.get || {}
+      {
+        schema_version: MODELS_SCHEMA_VERSION,
+        base_url: base,
+        responses_url: "#{base}/v1/responses",
+        default: default_model(settings),
+        fallbacks: fallback_models(settings),
+        utility_model: presence(settings["utility_model"]),
+        thinking_levels: ModelSelection::THINKING_LEVELS,
+        providers: providers,
+        agents: served_agents
+      }.compact
+    end
+
+    # The public docs index (name + title + fetchable raw-markdown url). Drives
+    # discovery: a coding agent lists this, then GETs the ones it needs.
+    def docs_index(base_url:)
+      base = normalize_base(base_url)
+      @docs.keys.sort.map do |slug|
+        { name: slug, title: doc_title(slug), url: "#{base}/docs/#{slug}.md" }
+      end
+    end
+
+    # Raw markdown for one public doc, by slug. nil = unknown slug (the transport 404s).
+    # No path traversal is possible: `slug` must be a KEY of the injected allowlist.
+    #
+    # The Jekyll frontmatter the docs site needs (title/parent/nav_order — the same
+    # files ARE the site's pages) is STRIPPED here: a coding agent asked for the prose,
+    # not for sidebar metadata, and this keeps the response byte-identical to what it
+    # was before the site existed.
+    def doc(slug)
+      read_doc(slug)&.sub(FRONTMATTER, "")
+    end
+
+    private
+
+    # Leading YAML block, plus the blank line after it, so the body still starts at
+    # its `# Heading`. Same shape SkillCatalog accepts.
+    FRONTMATTER = /\A---\s*\n(.*?)\n---\s*\n+/m
+    private_constant :FRONTMATTER
+
+    # Whole file, frontmatter included. nil = unknown slug or a file that vanished.
+    def read_doc(slug)
+      path = @docs[slug.to_s]
+      return nil if path.nil? || !File.file?(path)
+
+      File.read(path)
+    end
+
+    def substitute(text, base)
+      text
+        .gsub("{{BASE_URL}}", base)
+        .gsub("{{MODELS_URL}}", "#{base}/models.json")
+        .gsub("{{DOCS_URL}}", "#{base}/docs")
+    end
+
+    # default_model/default_provider -> { model:, provider: } | nil (nothing set).
+    def default_model(settings)
+      model = presence(settings["default_model"])
+      return nil if model.nil?
+
+      { model: model, provider: presence(settings["default_provider"]) }.compact
+    end
+
+    # Platform fallback chain as ["provider/model" | "model", …]; blank entries dropped.
+    def fallback_models(settings)
+      Array(settings["fallback_models"]).filter_map { |entry| presence(entry) }
+    end
+
+    # Providers a coding agent can target, from the MASKED store view: slug + the
+    # model ids. Deliberately NO base_url / auth / key — model ids and slugs are all
+    # that a `model`/`provider` line needs, and nothing here is a secret.
+    def providers
+      return nil if @provider_store.nil?
+
+      @provider_store.all.map do |rec|
+        { slug: rec["api"].to_s, models: Array(rec["models"]).map(&:to_s) }
+      end
+    end
+
+    # Agents already served by THIS insika — over the drop-in API the `model` field
+    # is the agent id, so this is the list of ids a client can call right now.
+    def served_agents
+      list = @agents.respond_to?(:call) ? Array(@agents.call) : nil
+      return nil if list.nil? || list.empty?
+
+      list.map do |a|
+        { id: a[:id].to_s, model: presence(a[:model]), provider: presence(a[:provider]),
+          description: presence(a[:description]) }.compact
+      end
+    end
+
+    # Frontmatter `title` (what the site's sidebar shows — one source for both
+    # surfaces), else the first markdown heading, else the humanized slug.
+    def doc_title(slug)
+      raw = read_doc(slug)
+      front = raw&.match(FRONTMATTER)
+      title = front && presence(Frontmatter.parse(front[1])["title"])
+      return title if title
+
+      heading = raw&.lines&.find { |l| l.start_with?("# ") }
+      heading ? heading.sub(/\A#\s*/, "").strip : slug.tr("-_", "  ").capitalize
+    end
+
+    def normalize_base(base_url) = base_url.to_s.sub(%r{/+\z}, "")
+    def presence(value) = Insika::Coercion.presence(value)
+  end
+end