insika 0.3.0 → 0.7.0

data/lib/insika/chat_builder.rb

@@ -15,7 +15,7 @@ module Insika
     def initialize(tool_registry:, skill_catalog:, checkpoint_store:, event_stream:,
                    hooks:, tool_catalog: nil, memory_store: nil, subagent_runner: nil,
                    tool_trace_store: nil, media_runner: nil, session_store: nil,
-                   contact_store: nil, followup_store: nil)
+                   contact_store: nil, followup_store: nil, knowledge_store: nil)
       @tool_registry = tool_registry
       @skill_catalog = skill_catalog
       @checkpoint_store = checkpoint_store
@@ -23,6 +23,10 @@ module Insika
       @hooks = hooks
       @tool_catalog = tool_catalog
       @memory_store = memory_store
+      # load_knowledge is the knowledge-read system tool — wired only with
+      # @knowledge_store present AND profile.knowledge["retrieve"] (a double
+      # gate, like remember). nil = never wired (parity).
+      @knowledge_store = knowledge_store
       # only to trace load_skill, which is not enveloped — nil = no trace (parity).
       @tool_trace_store = tool_trace_store
       # the object exposing #run_subagent (the Executor). nil = the
@@ -104,6 +108,15 @@ module Insika
                                      event_stream: @event_stream, state: state)
       end
 
+      # load_knowledge is a system default (outside the allowlist), like
+      # load_skill — wired only with @knowledge_store present AND the
+      # profile's opt-in (double gate, like remember). Reads the same
+      # (agent, tenant) scope the Knowledge context provider searches.
+      if @knowledge_store && Coercion.truthy?(state.profile.knowledge&.dig("retrieve"))
+        tools << Tools::LoadKnowledge.new(@knowledge_store, state.profile.id, tenant: state.tenant,
+                                          trace_recorder: @tool_trace_store, state: state)
+      end
+
       # update_briefing / set_next_step are the briefing-write system tools
       #  — wired only with @session_store present AND
       # profile.briefing_fields non-empty (double gate, like remember). Never
@@ -362,9 +375,16 @@ module Insika
         # under concurrency `after_tool_result` labelled every result with whichever
         # call started last.
         state.current_tool_name = tool_call.name.to_s
-        if state.current_tool_name == "load_skill"
+        case state.current_tool_name
+        when "load_skill"
           args = tool_call.arguments || {}
           emit.call(:skill_activated, { name: args["name"] || args[:name] })
+        when "load_knowledge"
+          # The adoption risk is the model never calling this tool, not what
+          # it injects — so the metric here is retrieval CALLS, tracked at
+          # the same point :skill_activated is.
+          args = tool_call.arguments || {}
+          emit.call(:knowledge_retrieved, { name: args["name"] || args[:name], agent: state.profile.id })
         else
           emit.call(:tool_call, { name: tool_call.name, arguments: tool_call.arguments })
         end

data/lib/insika/commands/agent_payload.rb

@@ -13,9 +13,9 @@ module Insika
                   tools_allow_groups skills skills_eager context_providers workflows_allow policies
                   prompt_refs limits approvals_required capabilities subagents tools_deferred
                   memory prompt_caching tool_persistence tool_output_compression budget reliability alerts
-                  routes stuck_signal outputs briefing_fields grounding funnel followup
+                  routes stuck_signal outputs stt_prompt briefing_fields grounding funnel followup
                   params model_policy guardrails refinement capabilities_declared
-                  edge_stream metadata distill harvest].freeze
+                  edge_stream metadata distill harvest knowledge schedules].freeze
 
       module_function
 

data/lib/insika/commands/backfill_knowledge.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  module Commands
+    # The recovery path (extraction is best-effort — a crash between a turn's
+    # terminal and the write loses that turn's concepts, not the conversation
+    # nor a previously learned one). Replays one agent's stored sessions
+    # through the SAME `Knowledge::Extractor` the per-turn hook uses.
+    #
+    # Synchronous — the CLI's own path (`insika knowledge:backfill`), same
+    # "no app boot" discipline as RunHarvest/RunDistillation: no task, no
+    # turn, nothing written but the concepts themselves.
+    class BackfillKnowledge
+      DEFAULT_MIN_MESSAGES = 2
+
+      def initialize(profiles:, knowledge_store:, session_store:, task_store:,
+                     settings_store: nil, event_stream:, extractor_factory: nil, consolidator_factory: nil)
+        @profiles = ProfileSource.coerce(profiles)
+        @knowledge_store = knowledge_store
+        @session_store = session_store
+        @task_store = task_store
+        @settings_store = settings_store
+        @extractor_factory = extractor_factory ||
+                             ->(config) { Knowledge::ExtractorFactory.build(config, utility_model: utility_model) }
+        @consolidator_factory = consolidator_factory ||
+                                ->(config) { Knowledge::ConsolidatorFactory.build(config, utility_model: utility_model) }
+        @event_stream = event_stream
+      end
+
+      # payload: { agent:, since?: ISO8601 }
+      # -> { backfilled: true, sessions: N, concepts: N, conflicts: N, dropped: {...} }
+      #  | { backfilled: false, skipped: "disabled|no_model|no_sessions" }
+      def call(command)
+        p = AgentPayload.symbolize(command.payload)
+        agent = AgentPayload.presence(p[:agent])
+        raise Insika::ValidationError, "agent is required" if agent.nil?
+
+        profile = @profiles[agent] || (raise Insika::NotFoundError, "agent '#{agent}' not configured")
+        config = Coercion.deep_stringify(profile.knowledge)
+        return skip("disabled") if config.nil? || !Coercion.truthy?(config["extract"])
+
+        extractor = @extractor_factory.call(config)
+        return skip("no_model") if extractor.nil?
+
+        # Same resolved model as the extractor. nil (unresolvable) is still
+        # meaningful: Knowledge.write_concept's conservative default.
+        consolidator = @consolidator_factory.call(config)
+
+        sessions = resolve_sessions(agent, p)
+        return skip("no_sessions") if sessions.empty?
+
+        concepts = 0
+        conflicts = 0
+        dropped = Hash.new(0)
+        sessions.each do |s|
+          result = extractor.extract(prompt: build_prompt(config, s[:messages]))
+          result[:concepts].each do |c|
+            case write_concept(profile, s[:id], c, consolidator)[:verdict]
+            when :new, :related then concepts += 1
+            when :contradicting then conflicts += 1
+            end
+          end
+          result[:dropped].each { |k, v| dropped[k] += v }
+        end
+
+        emit(:knowledge_backfilled, agent: agent, sessions: sessions.size, concepts: concepts,
+                                    conflicts: conflicts, dropped: dropped.to_h)
+        { backfilled: true, sessions: sessions.size, concepts: concepts, conflicts: conflicts, dropped: dropped.to_h }
+      end
+
+      private
+
+      def skip(reason) = { backfilled: false, skipped: reason }
+
+      # Every task of the agent, distinct session ids, newest first — the
+      # same reading `RunHarvest#window_session_ids` uses, without the
+      # last-N cap (a backfill is meant to be exhaustive over `--since`).
+      def resolve_sessions(agent, p)
+        since = AgentPayload.presence(p[:since])
+        tasks = @task_store.each_id.filter_map { |id| @task_store.find(id) }
+                           .select { |t| task_agent(t) == agent }
+                           .sort_by { |t| [t.created_at.to_s, t.id] }.reverse
+        tasks = tasks.select { |t| t.created_at.to_s >= since.to_s } if since
+
+        seen = {}
+        tasks.filter_map do |t|
+          sid = Coercion.presence(t.session_id)
+          next if sid.nil? || seen[sid]
+
+          seen[sid] = true
+          session = @session_store.find(sid)
+          next unless session && session.messages.size >= DEFAULT_MIN_MESSAGES
+
+          { id: sid.to_s, messages: session.messages }
+        end
+      end
+
+      def task_agent(task)
+        task.command.is_a?(Hash) ? task.command.dig("payload", "agent").to_s : ""
+      end
+
+      def build_prompt(config, messages)
+        base = Coercion.presence(config["prompt"]) || Knowledge::DEFAULT_PROMPT
+        <<~PROMPT
+          #{base.rstrip}
+
+          ## The conversation
+
+          #{render_transcript(messages)}
+        PROMPT
+      end
+
+      def render_transcript(messages)
+        redacted, = Insika::Safety::Detectors.redact(
+          messages.each_with_index.map { |m, i| "[#{i}] #{m['role']}: #{m['content']}" }.join("\n")
+        )
+        redacted
+      end
+
+      def write_concept(profile, session_id, concept, consolidator)
+        outcome = Knowledge.write_concept(store: @knowledge_store, agent_id: profile.id, concept: concept,
+                                          session_id: session_id, consolidator: consolidator)
+        case outcome[:verdict]
+        when :new, :related
+          emit(:knowledge_learned, name: outcome[:name], type: outcome[:type], agent: profile.id)
+        when :contradicting
+          emit(:knowledge_conflict, name: outcome[:name], agent: profile.id)
+        end
+        outcome
+      end
+
+      def utility_model
+        return nil unless @settings_store
+
+        @settings_store.get["utility_model"]
+      end
+
+      def emit(type, **data)
+        @event_stream.emit(Insika::Event.new(type: type, data: data, meta: { at: Time.now.utc.iso8601 }))
+      end
+    end
+  end
+end

data/lib/insika/commands/delete_artifact.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  module Commands
+    # The Studio's delete on the Artifacts tab — a bus command, like every
+    # Studio mutation (the Studio never writes a store directly). Idempotent:
+    # an unknown id is a no-op, not an error.
+    class DeleteArtifact
+      def initialize(artifact_store:, event_stream:)
+        @artifact_store = artifact_store
+        @event_stream = event_stream
+      end
+
+      # -> { deleted: id } | { deleted: nil } (unknown id).
+      def call(command)
+        id = Coercion.presence(command.payload[:id] || command.payload["id"])
+        raise ValidationError, "id is required" if id.nil?
+
+        deleted = @artifact_store.delete(id.to_s)
+        if deleted
+          @event_stream.emit(Insika::Event.new(
+                               type: :artifact_deleted,
+                               data: { id: id.to_s },
+                               meta: { at: Time.now.utc.iso8601 }
+                             ))
+          { deleted: id.to_s }
+        else
+          { deleted: nil }
+        end
+      end
+    end
+  end
+end

data/lib/insika/commands/delete_concept.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  module Commands
+    # Control command: removes a learned concept from the KnowledgeStore.
+    # -> { name, agent, tenant, deleted: true }.
+    class DeleteConcept
+      def initialize(knowledge_store:, event_stream:)
+        @knowledge_store = knowledge_store
+        @event_stream = event_stream
+      end
+
+      def call(command)
+        p = AgentPayload.symbolize(command.payload)
+        name = AgentPayload.presence(p[:name])
+        agent = AgentPayload.presence(p[:agent])
+        tenant = AgentPayload.presence(p[:tenant])
+        raise Insika::ValidationError, "name is required" if name.nil?
+        raise Insika::ValidationError, "agent is required" if agent.nil?
+
+        deleted = @knowledge_store.delete(agent, name, tenant: tenant)
+        raise Insika::NotFoundError, "concept '#{name}' not found for agent '#{agent}'" unless deleted
+
+        @event_stream.emit(Insika::Event.new(
+                             type: :knowledge_deleted, data: { name: name, agent: agent },
+                             meta: { at: Time.now.utc.iso8601 }
+                           ))
+        { name: name, agent: agent, tenant: tenant, deleted: true }
+      end
+    end
+  end
+end

data/lib/insika/commands/delete_mcp.rb

@@ -5,11 +5,14 @@ require "time"
 module Insika
   module Commands
     # Control command: removes an MCP instance from the
-    # McpStore. Idempotent (`existed: false` if there was none). -> { existed: bool }.
+    # McpStore. Idempotent (`existed: false` if there was none). Evicts any
+    # memoized client for this name — a removed instance stops answering
+    # tool calls right away, not after a restart. -> { existed: bool }.
     class DeleteMcp
-      def initialize(mcp_store:, event_stream:)
+      def initialize(mcp_store:, event_stream:, mcp_registry: nil)
         @mcp_store = mcp_store
         @event_stream = event_stream
+        @mcp_registry = mcp_registry
       end
 
       def call(command)
@@ -18,6 +21,7 @@ module Insika
         raise Insika::ValidationError, "name is required" if name.nil?
 
         existed = @mcp_store.delete(name)
+        @mcp_registry&.evict(name)
         @event_stream.emit(Insika::Event.new(
                              type: :mcp_deleted,
                              data: { name: name, existed: existed },

data/lib/insika/commands/delete_tenant_data.rb

@@ -21,9 +21,9 @@ module Insika
                      context_trace_store: nil, model_visible_trace_store: nil,
                      outcome_store: nil, task_store: nil,
                      checkpoint_store: nil, outbox_store: nil, shadow_pairs: nil,
-                     token_store: nil, funnel_store: nil, event_stream:,
+                      token_store: nil, funnel_store: nil, event_stream:,
                      followup_store: nil, contact_store: nil, proposal_store: nil,
-                     harvest_store: nil)
+                     harvest_store: nil, schedule_store: nil, artifact_store: nil)
         @memory_store = memory_store
         @token_store = token_store
         @session_store = session_store
@@ -40,6 +40,8 @@ module Insika
         @contact_store = contact_store   #  ; nil = nothing to sweep
         @proposal_store = proposal_store #  ; nil = nothing to sweep
         @harvest_store = harvest_store   #  ; nil = nothing to sweep
+        @schedule_store = schedule_store #  ; nil = nothing to sweep
+        @artifact_store = artifact_store #  ; nil = nothing to sweep
         @event_stream = event_stream
       end
 
@@ -72,6 +74,12 @@ module Insika
         # candidates reference sessions, and sessions carry the
         # tenant prefix — the harvest rows die with the tenant (D11).
         harvest = @harvest_store ? @harvest_store.purge(tenant: tenant) : 0
+        # recurring-schedule rows die with the tenant too (their
+        # message text is content, never kept behind an offboarded tenant).
+        schedules = @schedule_store ? @schedule_store.purge(tenant: tenant) : 0
+        # artifacts die with the tenant — a report is content, and the
+        # tenant binding is the isolation boundary (never a model-typed id).
+        artifacts = @artifact_store ? @artifact_store.purge(tenant: tenant) : 0
 
         @event_stream.emit(Insika::Event.new(
                              type: :tenant_data_deleted,
@@ -83,12 +91,16 @@ module Insika
                                      contacts: contacts,
                                      proposals: proposals,
                                      harvest: harvest,
+                                     schedules: schedules,
+                                     artifacts: artifacts,
                                      tokens_revoked: tokens_revoked }.merge(purged),
                              meta: { at: Time.now.utc.iso8601 }
                            ))
         { tenant: tenant, sessions: sessions, memory_records: memory_records,
           outcomes: outcomes, funnel: funnel, followups: followups, contacts: contacts,
-          proposals: proposals, harvest: harvest, tokens_revoked: tokens_revoked }.merge(purged)
+          proposals: proposals, harvest: harvest, schedules: schedules,
+          artifacts: artifacts,
+          tokens_revoked: tokens_revoked }.merge(purged)
       end
     end
   end

data/lib/insika/commands/gate_refinement.rb

@@ -12,7 +12,7 @@ module Insika
     # Synchronous like `run_refinement`, and for the same reason: it is operator-paced
     # work with a human waiting on the answer. It is NOT cheap — the replay is a real
     # conversation per golden case — so it is fired deliberately, never on a timer
-    # (the engine has no scheduler and this design does not add one).
+    # (a recurring timer may schedule OTHER turns; this gate stays human-paced).
     #
     # Payload:
     #   run_id     (required) a run in :completed — the evidence the candidate answers

data/lib/insika/commands/refresh_mcp_tools.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  module Commands
+    # Control command: LIVE re-discovery of an MCP instance's tools.
+    # Delegates to McpToolRegistry#refresh (connect +
+    # tools/list + write McpStore#tools_cache) — TOOL EXECUTION never reads
+    # that cache; this command only refreshes what `entries`/Studio/doctor
+    # display. The CLI verb / API route / Studio button that call it are
+    # later work (PR3/PR4) — the command itself is the seam they'll share.
+    # -> { instance:, tools: [{name,description,inputSchema}, ...] }.
+    class RefreshMcpTools
+      def initialize(mcp_registry:, event_stream:)
+        @mcp_registry = mcp_registry
+        @event_stream = event_stream
+      end
+
+      def call(command)
+        p = symbolize(command.payload)
+        name = Insika::Coercion.presence(p[:name]) ||
+               (raise Insika::ValidationError, "refresh_mcp_tools: 'name' (MCP instance) is required")
+
+        tools = @mcp_registry.refresh(name)
+        emit(name, tools)
+        { instance: name, tools: tools }
+      end
+
+      private
+
+      # Emits only the COUNT + the instance name (never a tool's schema, in
+      # case a schema description ever carries something sensitive): 0 leakage.
+      def emit(name, tools)
+        @event_stream.emit(Insika::Event.new(
+                             type: :mcp_tools_refreshed,
+                             data: { instance: name, tools: tools.size },
+                             meta: { at: Time.now.utc.iso8601 }
+                           ))
+      end
+
+      def symbolize(payload)
+        (payload || {}).each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
+      end
+    end
+  end
+end

data/lib/insika/commands/restore_concept.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  module Commands
+    # Control command: restores an old version of a concept as the current
+    # content (a new write — linear history, like `RestoreSystemFile`).
+    # -> { name, agent, tenant, updated_at }.
+    class RestoreConcept
+      def initialize(knowledge_store:, event_stream:)
+        @knowledge_store = knowledge_store
+        @event_stream = event_stream
+      end
+
+      def call(command)
+        p = AgentPayload.symbolize(command.payload)
+        name = AgentPayload.presence(p[:name])
+        agent = AgentPayload.presence(p[:agent])
+        tenant = AgentPayload.presence(p[:tenant])
+        raise Insika::ValidationError, "name is required" if name.nil?
+        raise Insika::ValidationError, "agent is required" if agent.nil?
+        raise Insika::ValidationError, "version is required" if p[:version].nil?
+
+        rec = @knowledge_store.restore(agent, name, p[:version], tenant: tenant)
+        @event_stream.emit(Insika::Event.new(
+                             type: :knowledge_learned, data: { name: name, agent: agent },
+                             meta: { at: Time.now.utc.iso8601 }
+                           ))
+        { name: name, agent: agent, tenant: tenant, updated_at: rec["updated_at"] }
+      end
+    end
+  end
+end

data/lib/insika/commands/seed_demo_data.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  module Commands
+    # Studio/CLI-facing adapter over Insika::Demo::Seeder — the one path both
+    # `insika demo:seed` and the Studio's "Seed demo data" button use (same
+    # discipline as RunDistillation: one object, two front doors).
+    class SeedDemoData
+      def initialize(seeder:, event_stream:)
+        @seeder = seeder
+        @event_stream = event_stream
+      end
+
+      # payload: { force: bool } -> the seeder's result Hash.
+      def call(command)
+        force = Coercion.truthy?(command.payload[:force] || command.payload["force"])
+        result = @seeder.seed!(force: force)
+        if result[:seeded]
+          @event_stream.emit(Insika::Event.new(
+                               type: :demo_data_seeded,
+                               data: { agent: result[:agent], counts: result[:counts] },
+                               meta: { at: Time.now.utc.iso8601 }
+                             ))
+        end
+        result
+      end
+    end
+  end
+end

data/lib/insika/commands/upsert_mcp.rb

@@ -8,17 +8,20 @@ module Insika
     # (transport/command/url/enabled + `env` credentials) in the McpStore. The
     # credentials are sentinel-aware per key (__OCULTO__ preserves; "" clears;
     # a new string replaces). Returns the MASKED record (env never comes back in
-    # plaintext). Durable config CRUD — running an MCP client against the
-    # instance is later runtime.
+    # plaintext). Evicts any memoized client for this name (mcp_registry is
+    # optional so a bare CLI/store-only caller need not wire one) so an edited
+    # command/url/env takes effect on the instance's next call — no restart.
     class UpsertMcp
-      def initialize(mcp_store:, event_stream:)
+      def initialize(mcp_store:, event_stream:, mcp_registry: nil)
         @mcp_store = mcp_store
         @event_stream = event_stream
+        @mcp_registry = mcp_registry
       end
 
       def call(command)
         p = AgentPayload.symbolize(command.payload)
         masked = @mcp_store.upsert(p) # validates `name`; returns masked
+        @mcp_registry&.evict(masked["name"])
 
         @event_stream.emit(Insika::Event.new(
                              type: :mcp_upserted,

data/lib/insika/commands/write_concept.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  module Commands
+    # Control command: writes a concept (complete concept markdown) into the
+    # KnowledgeStore. Validates the frontmatter (a `name` is required) before
+    # writing — same discipline as `WriteSkill`. The STORE POSITION (the
+    # `agent`/`name`/`tenant` triple) is the identity, exactly like a skill:
+    # an operator editing the raw markdown may leave the frontmatter `name:`
+    # unchanged even while promoting `provenance: observed` to `policy` —
+    # that promotion IS this command, no separate "promote" action needed.
+    # -> { name, agent, tenant, updated_at }.
+    class WriteConcept
+      def initialize(knowledge_store:, event_stream:)
+        @knowledge_store = knowledge_store
+        @event_stream = event_stream
+      end
+
+      def call(command)
+        p = AgentPayload.symbolize(command.payload)
+        name = AgentPayload.presence(p[:name])
+        agent = AgentPayload.presence(p[:agent])
+        tenant = AgentPayload.presence(p[:tenant])
+        content = p[:content].to_s
+        raise Insika::ValidationError, "name is required" if name.nil?
+        raise Insika::ValidationError, "agent is required" if agent.nil?
+
+        parsed = validate_frontmatter!(content)
+        rec = @knowledge_store.write(agent, name, content, tenant: tenant)
+        emit(parsed[:type], name, agent)
+        { name: name, agent: agent, tenant: tenant, updated_at: rec["updated_at"] }
+      end
+
+      private
+
+      def emit(type, name, agent)
+        @event_stream.emit(Insika::Event.new(
+                             type: :knowledge_learned, data: { name: name, type: type, agent: agent },
+                             meta: { at: Time.now.utc.iso8601 }
+                           ))
+      end
+
+      # Mirrors WriteSkill's frontmatter guard: without a parseable `name`,
+      # the concept would be unreachable by name once written (the Studio
+      # list keys on the store position, but the editor round-trips the
+      # content) — fail early, in the Command.
+      def validate_frontmatter!(content)
+        parsed = Insika::Knowledge::Concept.parse(content)
+        raise Insika::ValidationError, "concept missing YAML frontmatter (--- ... ---) or a `name`" if parsed.nil?
+
+        parsed
+      end
+    end
+  end
+end

data/lib/insika/context/priority.rb

@@ -20,6 +20,8 @@ module Insika
       PROMPT_REF   = 90  # Prompt Catalog guardrails/refs (Prompt) — pinned
       SKILL_BODY   = 85  # <active_skill> trigger-matched body (SkillTrigger)
       SKILL        = 80  # <available_skills> level 1 (Skill)
+      KNOWLEDGE    = 77  # <knowledge> top-K learned concepts (Knowledge) —
+                         #   below curated skills, above a single conversation's memory
       MEMORY       = 75  # <memory> read path (Memory)
       TOOL_SEARCH  = 70  # <available_tools> level 1 (ToolSearch)
       BRIEFING     = 65  # <briefing> session working state (Briefing) — D5: