insika 0.0.1 → 0.1.0

data/lib/insika/testing/store_contract.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+# Contract suite for Insika::Store (doc 01 §7), exported for third-party
+# backends (RFC-0018 A4) — a gem like `insika-pg` writes its spec against THIS
+# file, not against a read of stores/sqlite.rb:
+#
+#   require "insika/testing/store_contract"
+#
+#   RSpec.describe Insika::Stores::PG do
+#     subject(:store) { described_class.new(url:) }
+#     it_behaves_like "an Insika store"
+#   end
+#
+# Two groups, on purpose (RFC-0018 §4 item 4):
+#
+# - "an Insika store" — universal; every backend passes EXACTLY it (L2: the
+#   suite is honest — a test that passes on Memory passes on SQLite). The
+#   including group defines `store` (an empty, ready backend).
+# - "an Insika store safe for N workers" — OPT-IN; the multi-worker half. The
+#   including group ALSO defines `store_factory`, a callable returning ANOTHER
+#   connection to the SAME underlying backend. A backend that cannot serialize
+#   transactions across connections must not include it — Memory is the
+#   canonical example, and a single-connection backend has nothing to prove.
+#   Anything that wants to sit under `WEB_CONCURRENCY > 1` has to.
+#
+# Do not include backend-specific cases here (file durability, WAL, boot
+# races) — those belong to the backend's own spec.
+RSpec.shared_examples "an Insika store" do
+  describe "#get / #set (round-trip)" do
+    it "C1 preserves Hash with string keys" do # C1
+      store.set("s", "k", { "a" => 1, "b" => [1, 2] })
+      expect(store.get("s", "k")).to eq({ "a" => 1, "b" => [1, 2] })
+    end
+
+    it "C2 preserves Array" do # C2
+      store.set("s", "k", [1, "x", true, nil])
+      expect(store.get("s", "k")).to eq([1, "x", true, nil])
+    end
+
+    it "C3 preserves String" do # C3
+      store.set("s", "k", "texto")
+      expect(store.get("s", "k")).to eq("texto")
+    end
+
+    it "C4 preserves Integer" do # C4
+      store.set("s", "k", 42)
+      value = store.get("s", "k")
+      expect(value).to eq(42)
+      expect(value).to be_a(Integer)
+    end
+
+    it "C5 preserves Float" do # C5
+      store.set("s", "k", 3.14)
+      value = store.get("s", "k")
+      expect(value).to eq(3.14)
+      expect(value).to be_a(Float)
+    end
+
+    it "C6 preserves booleans" do # C6
+      store.set("s", "t", true)
+      store.set("s", "f", false)
+      expect(store.get("s", "t")).to be(true)
+      expect(store.get("s", "f")).to be(false)
+    end
+
+    it "C7 preserves nil written without exception" do # C7
+      store.set("s", "k", nil)
+      expect(store.get("s", "k")).to be_nil
+    end
+
+    it "C8 converts Symbols (keys and values) to Strings" do # C8
+      store.set("s", "k", { chave: :valor })
+      expect(store.get("s", "k")).to eq({ "chave" => "valor" })
+    end
+
+    it "C9 returns nil for an absent key, never an exception" do # C9
+      expect(store.get("s", "nao-existe")).to be_nil
+    end
+
+    it "C10 overwrites silently (last-write-wins)" do # C10
+      store.set("s", "k", "primeiro")
+      store.set("s", "k", "segundo")
+      expect(store.get("s", "k")).to eq("segundo")
+    end
+
+    it "C11 set returns the same object passed in (not the round-trip)" do # C11
+      obj = { "a" => 1 }
+      expect(store.set("s", "k", obj)).to equal(obj)
+    end
+  end
+
+  describe "#delete" do
+    it "C12 removes existing and returns true" do # C12
+      store.set("s", "k", 1)
+      expect(store.delete("s", "k")).to be(true)
+      expect(store.get("s", "k")).to be_nil
+    end
+
+    it "C13 returns false for a nonexistent key" do # C13
+      expect(store.delete("s", "k")).to be(false)
+    end
+  end
+
+  describe "#list" do
+    it "C14 returns scope keys sorted lexicographically" do # C14
+      store.set("s", "b", 1)
+      store.set("s", "a", 1)
+      store.set("s", "c", 1)
+      expect(store.list("s")).to eq(%w[a b c])
+    end
+
+    it "C15 filters by prefix with start_with? (not include?)" do # C15
+      store.set("s", "task:1", 1)
+      store.set("s", "task:2", 1)
+      store.set("s", "checkpoint:1", 1)
+      store.set("s", "my-task:1", 1) # trap: contains "task:" but does not start with it
+      expect(store.list("s", "task:")).to eq(%w[task:1 task:2])
+    end
+
+    it "C16 returns [] for an empty scope" do # C16
+      expect(store.list("s")).to eq([])
+    end
+
+    it "sorts lexicographically, not numerically" do # edge case 2
+      store.set("s", "task:10", 1)
+      store.set("s", "task:2", 1)
+      expect(store.list("s", "task:")).to eq(%w[task:10 task:2])
+    end
+  end
+
+  describe "scope isolation" do
+    it "C17 keeps scopes independent in get/list/delete" do # C17
+      store.set("s1", "k", 1)
+      store.set("s2", "k", 2)
+
+      expect(store.get("s1", "k")).to eq(1)
+      expect(store.get("s2", "k")).to eq(2)
+      expect(store.list("s1")).to eq(%w[k])
+
+      store.delete("s1", "k")
+      expect(store.get("s2", "k")).to eq(2)
+    end
+  end
+
+  describe "#transaction" do
+    it "C18 returns the block's value" do # C18
+      expect(store.transaction { 42 }).to eq(42)
+    end
+
+    it "C19 commits the block's writes" do # C19
+      store.transaction { store.set("s", "k", "commitado") }
+      expect(store.get("s", "k")).to eq("commitado")
+    end
+
+    it "C20 does a real rollback of set AND delete when the block raises" do # C20
+      store.set("s", "manter", "antigo")
+      store.set("s", "apagar", "existe")
+
+      expect do
+        store.transaction do
+          store.set("s", "manter", "novo")
+          store.delete("s", "apagar")
+          raise "boom"
+        end
+      end.to raise_error("boom")
+
+      # all the block's effects undone
+      expect(store.get("s", "manter")).to eq("antigo")
+      expect(store.get("s", "apagar")).to eq("existe")
+    end
+
+    it "C21 reuses the outer transaction when nested" do # C21
+      store.set("s", "k", "antigo")
+
+      expect do
+        store.transaction do
+          store.transaction { store.set("s", "k", "novo") }
+          raise "boom"
+        end
+      end.to raise_error("boom")
+
+      # outer rollback undoes the inner set (no nesting error)
+      expect(store.get("s", "k")).to eq("antigo")
+    end
+  end
+
+  describe "serialization errors" do
+    it "C22 raises StoreError and does not write a non-serializable value" do # C22
+      expect do
+        store.set("s", "k", Object.new)
+      end.to raise_error(Insika::StoreError)
+      expect(store.get("s", "k")).to be_nil
+    end
+  end
+end
+
+# Multi-worker safety (RFC-0018 A3). Every RFC-0016 A1 claim (outbox, delegation
+# sweep, recovery) is a read-check-write inside `transaction` — on SQLite that is
+# atomic because the backend opens BEGIN IMMEDIATE; a backend whose transaction
+# only yields passes all 22 cases above and still double-claims under two
+# workers, silently. These cases close that hole, across REAL concurrent
+# connections (threads, each on its own handle from `store_factory`).
+#
+# The `sleep` inside each transaction is deliberate: it holds the read state
+# open long enough that a backend without isolation ALWAYS lets a second
+# connection through, while a correct backend serializes the writers. Without
+# it a wrong backend could pass by scheduling luck — the failure E2 relies on.
+RSpec.shared_examples "an Insika store safe for N workers" do
+  # Starts N threads, each holding its OWN connection to the same backend,
+  # releases them together, and returns each block's value. Connections are
+  # closed on the way out when the backend has a #close.
+  def with_concurrent_connections(n, &blk)
+    ready = Queue.new
+    go = Queue.new
+    threads = n.times.map do
+      Thread.new do
+        conn = store_factory.call
+        ready << true
+        go.pop # hold every connection at the line, then release them together
+        blk.call(conn)
+      ensure
+        conn.close if conn.respond_to?(:close)
+      end
+    end
+    n.times { ready.pop }
+    go.close # a closed queue pops nil immediately: the starting gun
+    threads.map(&:value)
+  end
+
+  # The outbox/delegation claim, verbatim in shape: read the status, and only
+  # the connection that still sees "pending" may flip it. A correct backend
+  # serializes the transactions, so exactly ONE of the 8 observes "pending".
+  it "claims a pending key exactly once across concurrent connections" do
+    store.set("jobs", "job:1", "pending")
+
+    claims = with_concurrent_connections(8) do |conn|
+      conn.transaction do
+        next unless conn.get("jobs", "job:1") == "pending"
+
+        sleep 0.01 # hold the read open: a wrong backend lets everyone through
+        conn.set("jobs", "job:1", "claimed")
+        true
+      end
+    end.count(true)
+
+    expect(claims).to eq(1)
+    expect(store.get("jobs", "job:1")).to eq("claimed")
+  end
+
+  # The lost-update shape the claim generalizes from: read-modify-write of a
+  # counter. A backend whose transaction yields without isolation drops
+  # increments whenever two connections overlap.
+  it "does not lose updates across concurrent connections" do
+    store.set("meters", "hits", 0)
+    connections = 4
+    increments = 10
+
+    with_concurrent_connections(connections) do |conn|
+      increments.times do
+        conn.transaction do
+          current = conn.get("meters", "hits")
+          sleep 0.001 # widen the window a wrong backend loses updates through
+          conn.set("meters", "hits", current + 1)
+        end
+      end
+    end
+
+    expect(store.get("meters", "hits")).to eq(connections * increments)
+  end
+end

data/lib/insika/token_estimator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Insika
+  # Cheap token estimate behind an interface.
+  # Default: text.length / 4 (off by ~±15%, absorbed by the budget margin).
+  # Swappable for a real tokenizer without touching the Builder: any object that
+  # responds to #estimate(text) -> Integer works as a replacement (injected at the
+  # composition root).
+  module TokenEstimator
+    module_function
+
+    def estimate(text)
+      text.to_s.length.ceildiv(4)
+    end
+  end
+end

data/lib/insika/tool_assembly.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require "async"
+require "async/semaphore"
+
+module Insika
+  # Turn-scoped assembly of the agent's tool instances (pipeline stage 3 tail):
+  # capability resolution, instantiation (Entry#factory | ready instance),
+  # turn-context (D2) injection, the capability<->direct dedup join, and the
+  # ToolEnvelope wrap (stage 7 seam: per-call timeout + side-effect recording).
+  #
+  # Extracted from the Executor (§11 B5) to keep the hot-path file smaller. It is
+  # a pure collaborator — it holds only the injected registries/stores and no
+  # per-turn state; everything turn-specific arrives via `state`/`turn_context`.
+  # The Executor keeps thin delegators (resolve_capabilities/assemble_tool_instances/
+  # wrap_tools) so the existing private-method contract stays intact.
+  class ToolAssembly
+    def initialize(tool_registry:, capability_registry:, event_stream:,
+                   checkpoint_store:, tool_trace_store:)
+      @tool_registry = tool_registry
+      @capability_registry = capability_registry
+      @event_stream = event_stream
+      @checkpoint_store = checkpoint_store
+      @tool_trace_store = tool_trace_store
+    end
+
+    # Resolution sub-step BETWEEN Context and Policy —
+    # it does NOT feed candidate_tools (those stay ONLY tool_registry.entries,
+    # a capability does not go through the ToolAllowlist). Resolves each
+    # capability of the profile to the concrete Entry already registered in the
+    # tool_registry and keeps the impl_name -> capability_name mapping for the
+    # post-Policy join. Errors
+    # (Unavailable/Ambiguous, or an unregistered impl) propagate as a
+    # CapabilityError -> single capture in `execute` (stage :capability). Without
+    # @capability_registry OR without profile.capabilities: {} (parity).
+    def resolve_capabilities(profile, context)
+      return {} if @capability_registry.nil?
+
+      Array(profile.capabilities).each_with_object({}) do |cap_name, names|
+        provider = @capability_registry.resolve(cap_name, profile: profile, context: context,
+                                                           event_stream: @event_stream)
+        next if provider.kind == :workflow # exposure to the agent loop is a follow-up
+
+        entry = @tool_registry.entries.find { |e| e.name == provider.impl_name.to_s }
+        if entry.nil?
+          raise CapabilityError, "capability '#{cap_name}' resolveu para impl " \
+                                 "'#{provider.impl_name}', not registered in tool_registry"
+        end
+
+        names[entry.name] ||= cap_name.to_s # the 1st capability to claim an impl wins
+      end
+    end
+
+    # Joins the direct instances (Policy/ToolAllowlist) with the
+    # capability-sourced ones (grant = profile.capabilities — they never went
+    # through Policy). Avoids double-exposure: if the SAME impl_name was also
+    # allowed directly, the DIRECT instance is discarded — the model sees only the
+    # capability alias.
+    def assemble_tool_instances(allowed, state)
+      names = state.respond_to?(:capability_names) ? (state.capability_names || {}) : {}
+      ctx = state.respond_to?(:turn_context) ? state.turn_context : nil
+      return instantiate_tools(allowed, ctx) if names.empty?
+
+      # Dedup by the ENTRY NAME (registry key = impl_name) BEFORE
+      # instantiating — the INSTANCE's `.name` (RubyLLM) is not the registration
+      # name.
+      direct = Array(allowed).reject { |e| e.respond_to?(:name) && names.key?(e.name.to_s) }
+      instantiate_tools(direct, ctx) + capability_tool_instances(names, ctx)
+    end
+
+    # Envelopes each allowed tool (per-call timeout + side-effect recording).
+    # The system LoadSkill (configure_chat) is NOT enveloped — it is a system
+    # tool with no side-effect and of trivial latency.
+    def wrap_tools(tools, state, skip_side_effects = [])
+      timeout = state.profile.limits[:tool_timeout] || 60
+      install_tool_gate(state)
+      tools.map do |tool|
+        ToolEnvelope.new(tool, state: state, checkpoint_store: @checkpoint_store,
+                               tool_registry: @tool_registry, timeout: timeout,
+                               skip_side_effects: skip_side_effects,
+                               trace_recorder: @tool_trace_store)
+      end
+    end
+
+    private
+
+    # D4 (item 30): the model decides the fan-out, so without a cap a batch of 15
+    # data-tools is 15 simultaneous requests to the same upstream — which is how
+    # one turn earns a 429 for every other turn in the process. Note the contrast
+    # with the primitives that already shipped: `spawn_subagents` caps at 8
+    # (SubagentGraph.fan_out_cap) and `Tools::Concurrency.gather` takes `max:`.
+    #
+    # The cap is enforced in OUR decorator, not in the gem (which has none): ONE
+    # semaphore per turn, installed here — once, in the turn's own fiber, before
+    # any tool can run — and acquired by every ToolEnvelope. Off (nil) unless the
+    # turn actually got concurrency, so the serial path allocates nothing.
+    # A state that predates these fields (a unit stub) is left alone.
+    def install_tool_gate(state)
+      return unless state.respond_to?(:tool_gate) && state.respond_to?(:tool_concurrency)
+
+      cap = state.tool_concurrency
+      state.tool_gate = cap ? Async::Semaphore.new(cap) : nil
+    end
+
+    # Real Engine -> Entries (respond to factory); fakes -> ready instances.
+    # `turn_context` (D2) is deposited into the instances that expose it
+    # (data-tools); the rest ignore it (parity).
+    def instantiate_tools(allowed, turn_context = nil)
+      Array(allowed).map do |t|
+        tool = t.respond_to?(:factory) ? t.factory.call : t
+        inject_turn_context(tool, turn_context)
+        tool
+      end
+    end
+
+    # D2/G3 seam: deposits the turn context into the freshly created instance
+    # (same idea as `remember`, which receives tenant/state) BEFORE the
+    # ToolEnvelope. Duck-typed: only what exposes `turn_context=` (DataDefinedTool)
+    # receives it. nil (a state with no turn_context, e.g. a test stub) -> no-op.
+    def inject_turn_context(tool, turn_context)
+      return if turn_context.nil?
+
+      tool.turn_context = turn_context if tool.respond_to?(:turn_context=)
+    end
+
+    # impl_name -> Capability::ResolvedTool(capability_name:), STILL without
+    # ToolEnvelope (the call site's wrap_tools wraps the whole set — same
+    # order impl -> ResolvedTool -> ToolEnvelope). entry already validated in
+    # resolve_capabilities.
+    def capability_tool_instances(names, turn_context = nil)
+      names.map do |impl_name, capability_name|
+        entry = @tool_registry.entries.find { |e| e.name == impl_name }
+        tool = entry.factory.call
+        inject_turn_context(tool, turn_context)
+        Capability::ResolvedTool.new(tool, capability_name: capability_name,
+                                           impl_name: impl_name)
+      end
+    end
+  end
+end

data/lib/insika/tool_catalog.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Insika
+  # "Level 1" (progressive disclosure) view of the tools: just
+  # name+description, the analog of SkillCatalog for tools. It does NOT read
+  # disk (the tools are already in the ToolRegistry, registered at boot) and does NOT
+  # inherit from RubyLLM::Tool — duck typing over `.description` (pure, testable without
+  # the gem).
+  #
+  # The canonical `description` does not live in the Entry (Registry::Entry has no such
+  # field): it comes from the tool INSTANCE (`factory.call.description`). That is why the
+  # catalog is LAZY: it only instantiates the tools on the first query (`all`), not at
+  # boot. A deployment without `tools_deferred` never touches the catalog and pays for
+  # no instantiation at all; a broken factory surfaces on first use (where the
+  # Executor would also catch it at stage 3), not at construction.
+  class ToolCatalog
+    Entry = Data.define(:name, :description)
+
+    def initialize(tool_registry:)
+      @tool_registry = tool_registry
+    end
+
+    def all
+      @entries ||= build_entries
+    end
+
+    # Reloads the index (after authoring a data-tool in the overlay). Mirrors
+    # SkillCatalog#reload — level-1/tool_search starts seeing the new tool without a
+    # restart. An in-flight turn has already captured `all`.
+    def reload
+      @entries = build_entries
+      self
+    end
+
+    # Allowed deferred slice (typically allowed_tools ∩ tools_deferred).
+    # Names outside the catalog are silently ignored (safe failure: less
+    # exposure, never more).
+    def subset(names)
+      wanted = Array(names).map(&:to_s)
+      all.select { |e| wanted.include?(e.name) }
+    end
+
+    # PURE matcher: case-insensitive, substring/keyword, NO
+    # embeddings. name weighs 2, description weighs 1; ties broken by original index
+    # (Ruby's sort_by is not stable). `within:` restricts the universe via subset.
+    def search(query, within: nil)
+      terms = query.to_s.downcase.split(/\s+/).reject(&:empty?)
+      return [] if terms.empty?
+
+      universe = within ? subset(within) : all
+      scored = universe.each_with_index.filter_map do |entry, idx|
+        score = score_entry(entry, terms)
+        [entry, score, idx] if score.positive?
+      end
+      scored.sort_by { |_entry, score, idx| [-score, idx] }.map(&:first)
+    end
+
+    # Level 1 injected into the prompt — mirror of SkillCatalog#format_for_prompt,
+    # swapping the tag and the final instruction (load_skill -> tool_search).
+    def format_for_prompt(entries = all)
+      return "" if entries.empty?
+
+      lines = entries.map { |e| %(  <tool name="#{e.name}">#{e.description}</tool>) }.join("\n")
+
+      <<~PROMPT.strip
+        <available_tools>
+        #{lines}
+        </available_tools>
+
+        Before using a tool above, call `tool_search` with what you need
+        to do to enable it in this conversation.
+      PROMPT
+    end
+
+    private
+
+    def build_entries
+      @tool_registry.entries.map do |entry|
+        Entry.new(name: entry.name, description: entry.factory.call.description.to_s)
+      end
+    end
+
+    def score_entry(entry, terms)
+      name = entry.name.downcase
+      desc = entry.description.downcase
+      terms.sum { |term| (name.include?(term) ? 2 : 0) + (desc.include?(term) ? 1 : 0) }
+    end
+  end
+end