insika 0.2.0 → 0.7.0

data/lib/insika/commands/rollback_harvest.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "time"
+require "securerandom"
+
+module Insika
+  module Commands
+    #   — the snapshot restored. `RollbackHarvest(snapshot_ref:)`
+    # restores the pre-promotion state deterministically: WriteSkill with the
+    # snapshot's pre-promotion content (or DeleteSkill when the skill did not
+    # exist before — the snapshot's `existed` flag), SetSkillAgents to the
+    # snapshot's pre-promotion `enabled_for`, and the `rolled_back_at` stamp on
+    # the SAME log (the log stays the single ledger: a skill promoted, rolled
+    # back, re-promoted is three readable rows). The "rehearsed once
+    # end-to-end" acceptance is this command's full promote -> rollback cycle.
+    class RollbackHarvest
+      def initialize(harvest_store:, skill_store:, skill_catalog:, profile_source:,
+                     event_stream:, write_skill: nil, delete_skill: nil, set_skill_agents: nil)
+        @harvest_store = harvest_store
+        @profiles = ProfileSource.coerce(profile_source)
+        @event_stream = event_stream
+        @write_skill = write_skill ||
+                       Insika::Commands::WriteSkill.new(skill_store: skill_store,
+                                                        skill_catalog: skill_catalog,
+                                                        event_stream: event_stream)
+        @delete_skill = delete_skill ||
+                        Insika::Commands::DeleteSkill.new(skill_store: skill_store,
+                                                          skill_catalog: skill_catalog,
+                                                          event_stream: event_stream)
+        @set_skill_agents = set_skill_agents ||
+                            Insika::Commands::SetSkillAgents.new(profile_source: @profiles,
+                                                                 event_stream: event_stream)
+      end
+
+      # payload: { snapshot_ref:, operator?, reason? }
+      # -> Promotion (the row with rolled_back_at stamped)
+      def call(command)
+        p = AgentPayload.symbolize(command.payload)
+        snapshot_ref = AgentPayload.presence(p[:snapshot_ref])
+        raise Insika::ValidationError, "snapshot_ref is required" if snapshot_ref.nil?
+
+        snap = @harvest_store.find_snapshot(snapshot_ref) ||
+               (raise Insika::NotFoundError, "harvest snapshot not found: #{snapshot_ref}")
+        # The log row whose snapshot_ref matches — but a promotion that never
+        # recorded its row (a crash between the writes and the append) rolls
+        # back by snapshot, and the rollback stamp catches up on whatever row
+        # exists (D8's honest price, resolved by reference).
+        promotion = promotion_for_snapshot(snap)
+        operator = (AgentPayload.presence(p[:operator]) || command.meta[:operator] || "operator").to_s
+
+        if snap.existed
+          @write_skill.call(Insika::Command.build(:write_skill,
+                                                  { name: snap.skill, content: snap.content,
+                                                    agent: snap.agent }))
+        else
+          begin
+            @delete_skill.call(Insika::Command.build(:delete_skill,
+                                                     { name: snap.skill, agent: snap.agent }))
+          rescue Insika::NotFoundError
+            # the desired state (gone) already holds — a snapshot taken for a
+            # skill that never landed rolls back cleanly
+          end
+        end
+        @set_skill_agents.call(Insika::Command.build(:set_skill_agents,
+                                                     { name: snap.skill,
+                                                       agent_ids: snap.enabled_for }))
+
+        rolled = promotion ? @harvest_store.append_rollback(promotion_id: promotion.id,
+                                                    operator: operator, reason: p[:reason]) : nil
+        @event_stream.emit(Insika::Event.new(
+                             type: :skill_rolled_back,
+                             data: { snapshot_ref: snapshot_ref, skill: snap.skill,
+                                     agent: snap.agent, operator: operator }.compact,
+                             meta: { at: Time.now.utc.iso8601 }
+                           ))
+        rolled || promotion
+      end
+
+      private
+
+      def promotion_for_snapshot(snap)
+        @harvest_store.promotions.find { |row| row.snapshot_ref == snap.id }
+      end
+    end
+  end
+end

data/lib/insika/commands/run_distillation.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module Insika
+  module Commands
+    # the ONLY path that writes proposals. Distills ONE session
+    # end to end: read the transcript and the memory baseline, ask the model,
+    # schema-drop, dedup against the ledger, write proposals, mark the session
+    # distilled. Synchronous (it runs on the engine's worker fiber, C5) — it
+    # creates no task and no turn (D2: no TaskStore work unit — a crash
+    # mid-pass leaves the marker unwritten and the next pass re-scans; a
+    # duplicate proposal is filtered by the ledger, never applied).
+    class RunDistillation
+      DEFAULT_IDLE_HOURS = 6
+      DEFAULT_MIN_MESSAGES = 3
+      DEFAULT_MAX_PROPOSALS = 10
+
+      def initialize(profiles:, proposal_store:, session_store:, memory_store:,
+                     settings_store:, event_stream:, distiller_factory: nil)
+        @profiles = profiles
+        @proposal_store = proposal_store
+        @session_store = session_store
+        @memory_store = memory_store
+        @settings_store = settings_store
+        @event_stream = event_stream
+        @distiller_factory = distiller_factory ||
+                             ->(config) { Distill::DistillerFactory.build(config, utility_model: utility_model) }
+      end
+
+      # payload: { session_id: } (agent resolved from the session, D6-bis)
+      # -> { distilled: true, proposals: N, dropped: {...}, deduped: N, cost: {...} | nil }
+      #  | { distilled: false, skipped: "already|untagged|no_agent|disabled|
+      #        too_fresh|too_short|no_model" }
+      def call(command)
+        session_id = Coercion.presence(command.payload[:session_id] || command.payload["session_id"])
+        raise ValidationError, "session_id is required" if session_id.nil?
+
+        session = @session_store.find(session_id)
+        raise Insika::NotFoundError, "session not found: #{session_id}" if session.nil?
+
+        return skip("already") if @proposal_store.distilled?(session_id)
+
+        customer = Coercion.presence(session.vars["customer"])
+        return skip("untagged") if customer.nil?
+
+        agent_id = Coercion.presence(session.vars["agent"])
+        return skip("no_agent") if agent_id.nil?
+
+        profile = @profiles[agent_id]
+        return skip("no_agent") if profile.nil?
+
+        config = Coercion.deep_stringify(profile.distill)
+        return skip("disabled") if config.nil? || !Coercion.truthy?(config["enabled"])
+
+        # D4: the pack's own idle threshold wins over the scan's lower bound —
+        # a pack that wants 12 h is never distilled at 6.
+        idle_hours = config["idle_hours"].to_i
+        idle_hours = DEFAULT_IDLE_HOURS unless idle_hours.positive?
+        return skip("too_fresh") unless idle?(session.updated_at, idle_hours)
+
+        min_messages = config["min_messages"].to_i
+        min_messages = DEFAULT_MIN_MESSAGES unless min_messages.positive?
+        return skip("too_short") if session.messages.size < min_messages
+
+        tenant = tenant_of(session_id)
+        distiller = @distiller_factory.call(config)
+        return skip("no_model") if distiller.nil?
+
+        baseline = memory_baseline(tenant, customer)
+        prompt = build_prompt(config, session, baseline)
+        result = distiller.distill(prompt: prompt, message_count: session.messages.size,
+                                   max_proposals: (config["max_proposals"] || DEFAULT_MAX_PROPOSALS).to_i)
+
+        deduped = 0
+        survivors = result[:proposals].reject do |proposal|
+          dedup = deduped?(tenant, customer, proposal, baseline)
+          deduped += 1 if dedup
+          dedup
+        end
+
+        survivors.each do |proposal|
+          expected = baseline[proposal["name"]]
+          @proposal_store.create(
+            tenant: tenant, customer: customer, session_ref: session_id,
+            key: proposal["name"], value: proposal["value"],
+            confidence: proposal["confidence"], evidence: proposal["turns"],
+            expected_revision: expected && expected.updated_at,
+            expected_existed: !expected.nil?
+          )
+        end
+
+        @proposal_store.mark_distilled(session_id, agent: agent_id,
+                                       proposals: survivors.size,
+                                       dropped: result[:dropped],
+                                       deduped: deduped,
+                                       cost: result[:cost])
+        @event_stream.emit(Insika::Event.new(
+                             type: :distillation_completed,
+                             # counts and ids only (D7) — a fact value never
+                             # enters the stream.
+                             data: { session_ref: session_id, agent: agent_id,
+                                     proposals: survivors.size,
+                                     dropped: result[:dropped],
+                                     deduped: deduped,
+                                     cost: result[:cost] },
+                             meta: { at: Time.now.utc.iso8601 }
+                           ))
+        { distilled: true, proposals: survivors.size, dropped: result[:dropped],
+          deduped: deduped, cost: result[:cost] }
+      end
+
+      private
+
+      # nil tenant in single-tenant deployments; the "<tenant>:" prefix of the
+      # session id otherwise (D1 — the same reading ForgetCustomer uses).
+      def tenant_of(session_id)
+        tenant, rest = session_id.to_s.split(":", 2)
+        rest.nil? ? nil : tenant
+      end
+
+      def skip(reason)
+        { distilled: false, skipped: reason }
+      end
+
+      def idle?(updated_at, idle_hours)
+        return false if Coercion.blank?(updated_at)
+
+        Time.iso8601(updated_at.to_s) <= Time.now.utc - idle_hours * 3600
+      rescue ArgumentError
+        false
+      end
+
+      # Per fact key: the existence + updated_at at distill time (D5 — the CAS
+      # baseline the approval compares against). Sorted by key for a stable
+      # prompt.
+      def memory_baseline(tenant, customer)
+        @memory_store.facts(tenant: tenant, customer: customer)
+                     .each_with_object({}) { |f, acc| acc[f.key] = f }
+      end
+
+      def deduped?(tenant, customer, proposal, baseline)
+        key = proposal["name"].to_s
+        value = proposal["value"].to_s
+        return true if @proposal_store.decided?(tenant: tenant, customer: customer,
+                                                key: key, value: value)
+        return true if @proposal_store.open_pending?(tenant: tenant, customer: customer, key: key)
+
+        applied = baseline[key]
+        !applied.nil? && applied.value == value && applied.origin.to_s.start_with?("distilled:")
+      end
+
+      # The prompt gets the transcript slice (masked through the
+      # output filter first — the   redaction rule), the
+      # customer's CURRENT facts (so the model can avoid re-proposing applied
+      # facts), and the answer rules (the pack prompt or DEFAULT_PROMPT).
+      def build_prompt(config, session, baseline)
+        base = Coercion.presence(config["prompt"]) || Distill::DEFAULT_PROMPT
+        transcript = render_transcript(session.messages)
+        facts = baseline.values.map { |f| "- #{f.key}: #{f.value}" }
+        <<~PROMPT
+          #{base.rstrip}
+
+          ## The conversation
+
+          #{transcript}
+
+          ## Facts already in this customer's memory (do not re-propose them)
+
+          #{facts.empty? ? "(none)" : facts.join("\n")}
+        PROMPT
+      end
+
+      def render_transcript(messages)
+        redacted, = Insika::Safety::Detectors.redact(
+          messages.each_with_index.map { |m, i| "[#{i}] #{m['role']}: #{m['content']}" }.join("\n")
+        )
+        redacted
+      end
+
+      def utility_model
+        return nil unless @settings_store
+
+        @settings_store.get["utility_model"]
+      end
+    end
+  end
+end

data/lib/insika/commands/run_harvest.rb

@@ -0,0 +1,393 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  module Commands
+    # the ONLY path that writes candidates. Mines ONE window end
+    # to end: resolve the sessions, read the transcripts + evidence, ask the
+    # miner, schema-drop, apply the negative list and the grounding filter,
+    # dedup against the ledger, write the run + candidates, stamp the markers.
+    #
+    # Synchronous (it runs on the engine's worker fiber, C12, or the CLI, C15)
+    # — it creates no task and no turn. It writes NOTHING to sessions or
+    # skills: D2's fork discipline is this command's contract — a customer
+    # turn's prefix is untouched by construction, and the only harvest-side
+    # spend is the run's cost (E1).
+    #
+    # Payload: { agent:, last_sessions?, since?, full?, session_ids?,
+    #            max_proposals?, exclude_sessions? }
+    class RunHarvest
+      DEFAULT_LAST_SESSIONS = 200 # the EvidenceCollector's window default
+      DEFAULT_MIN_MESSAGES = 3
+      DEFAULT_MAX_PROPOSALS = 10
+      DEFAULT_IDLE_HOURS = 24
+
+      def initialize(profiles:, harvest_store:, session_store:, task_store:,
+                     skill_store: nil, tool_trace_store: nil, settings_store: nil,
+                     negative_list: nil, miner_factory: nil, event_stream:)
+        @profiles = ProfileSource.coerce(profiles)
+        @harvest_store = harvest_store
+        @session_store = session_store
+        @task_store = task_store
+        @skill_store = skill_store
+        @tool_trace_store = tool_trace_store
+        @settings_store = settings_store
+        @negative_list = negative_list
+        @miner_factory = miner_factory ||
+                         ->(config) { Harvest::MinerFactory.build(config, utility_model: utility_model) }
+        @event_stream = event_stream
+      end
+
+      # -> { mined: true, run_id:, candidates: N,
+      #      rejected: { "<rule-id>" => N, "ungrounded" => N, "dedup" => N,
+      #                  "schema" => N, ... }, cost: {...} | nil }
+      #  | { mined: false, skipped: "disabled|no_model|no_grounding_matcher" }
+      def call(command)
+        p = AgentPayload.symbolize(command.payload)
+        agent = AgentPayload.presence(p[:agent])
+        raise Insika::ValidationError, "agent is required" if agent.nil?
+
+        profile = @profiles[agent] ||
+                  (raise Insika::NotFoundError, "agent '#{agent}' not configured")
+        config = Coercion.deep_stringify(profile.harvest)
+        return skip("disabled") if config.nil? || !Coercion.truthy?(config["enabled"])
+
+        # D4: the OPERATIVE negative list lives on the profile (hot-editable,
+        # seeded by `insika harvest:negative import`); the injected list is
+        # the deployment's fallback. The engine applies data, never authors it.
+        @list = negative_list_for(config)
+
+        # D3: product claims cannot be verified without a matcher, so NOTHING
+        # mines — refused, not warned (D12's "by refusal, not by prompt").
+        grounding = Grounding.parse(profile.grounding)
+        return skip("no_grounding_matcher") if grounding.nil? || !grounding.matcher.sku?
+
+        miner = @miner_factory.call(config)
+        return skip("no_model") if miner.nil?
+
+        run = @harvest_store.create_run(agent_id: agent, window: window_record(p),
+                                        budget: budget_cap(config))
+        sessions = resolve_sessions(agent, p, config)
+
+        begin
+          # No eligible sessions: a run that says "we looked and it was clean"
+          # without paying a bill; markers untouched (re-scan, D10).
+          if sessions.empty?
+            @harvest_store.complete_run(run.id, candidates: 0)
+            return { mined: true, run_id: run.id, candidates: 0,
+                     rejected: empty_rejected, cost: nil }
+          end
+
+          prompt = build_prompt(config, sessions, agent)
+          result = miner.mine(prompt: prompt,
+                              message_counts: sessions.map { |s| s[:messages].size },
+                              max_proposals: max_proposals(p, config))
+
+          # The mining budget is a REAL cap (the review fix): a pass that
+          # spent more than the pack declared is failed with the numbers and
+          # proposes NOTHING — the docs' "the budget cap bounds it" is
+          # enforced here, post-hoc for the one model call, pre-hoc for every
+          # downstream write and gate.
+          if budget_exceeded?(budget_cap(config), result[:cost])
+            @harvest_store.fail_run(run.id, error: "mining budget exceeded: " \
+                                                 "spent #{result[:cost]['spent']} > " \
+                                                 "#{budget_cap(config)['tokens']}")
+            emit(:harvest_mined, agent: agent, run_id: run.id, candidates: 0,
+                                 rejected: empty_rejected, cost: result[:cost])
+            return { mined: true, run_id: run.id, candidates: 0,
+                     rejected: empty_rejected, cost: result[:cost] }
+          end
+
+          survivors, rejected = filter_skills(result[:skills], sessions, agent, grounding.matcher)
+
+          survivors.each do |skill|
+            @harvest_store.create_candidate(
+              run_id: run.id, agent: agent, name: skill["name"],
+              description: skill["description"], body: skill_md(skill),
+              triggers: skill["triggers"] || [], rationale: skill["rationale"].to_s,
+              origin: sessions.map { |s| s[:id] }, evidence_turns: skill["evidence_turns"] || [],
+              proposer: miner.model
+            )
+          end
+
+          final_rejected = merge_rejected(result[:dropped], rejected)
+          @harvest_store.complete_run(run.id, candidates: survivors.size,
+                                              cost: result[:cost],
+                                              rejected: final_rejected)
+          # Markers AFTER the pass completes (D10's crash-safe re-scan).
+          sessions.each { |s| @harvest_store.mark_mined(s[:id], candidates: survivors.size) }
+
+          emit(:harvest_mined, agent: agent, run_id: run.id, candidates: survivors.size,
+                               rejected: final_rejected, cost: result[:cost])
+          { mined: true, run_id: run.id, candidates: survivors.size,
+            rejected: final_rejected, cost: result[:cost] }
+        rescue StandardError => e
+          # The run is failed; the markers are NOT written (re-scan, D10);
+          # the exception propagates to the caller's fiber or the CLI.
+          begin
+            @harvest_store.fail_run(run.id, error: e.message)
+          rescue StandardError
+            nil
+          end
+          raise
+        end
+      end
+
+      private
+
+      def skip(reason)
+        { mined: false, skipped: reason }
+      end
+
+      # The pack's harvest.miner.budget as { "tokens" => N } — nil when
+      # absent/not positive (no cap, the refinement discipline).
+      def budget_cap(config)
+        raw = config && config.dig("miner", "budget")
+        raw.is_a?(Hash) && raw["tokens"].to_i.positive? ? { "tokens" => raw["tokens"].to_i } : nil
+      end
+
+      def budget_exceeded?(cap, cost)
+        cap && cost && cost["spent"].to_i > cap["tokens"].to_i
+      end
+
+      # The candidate's body is a full SKILL.md by construction: the model
+      # writes the procedure, the engine wraps the frontmatter (name /
+      # description / triggers — the miner's own fields), so a candidate is
+      # servable the moment a human promotes it (WriteSkill validates the
+      # frontmatter, and the gate's clone serves it the same way).
+      def skill_md(skill)
+        fm = { "name" => skill["name"], "description" => skill["description"] }
+        fm["triggers"] = Array(skill["triggers"]).join(", ") if Array(skill["triggers"]).any?
+        "---\n#{fm.map { |k, v| "#{k}: #{v}" }.join("\n")}\n---\n#{skill['body']}"
+      end
+
+      # The profile's own rules win over the injected seed (D4). A malformed
+      # profile list parses to nil — the injected list stays the fallback,
+      # never a silent empty.
+      def negative_list_for(config)
+        return @negative_list unless config && config["negative_list"]
+
+        Harvest::NegativeList.parse(config["negative_list"]) || @negative_list
+      end
+
+      def empty_rejected = { "schema" => 0, "unknown_key" => 0, "oversized" => 0,
+                             "bad_turns" => 0, "duplicate" => 0, "capped" => 0 }
+
+      def window_record(p)
+        return { "session_ids" => Array(p[:session_ids]).map(&:to_s) } if p[:session_ids]
+        return { "since" => p[:since].to_s } if AgentPayload.presence(p[:since])
+        return { "last_sessions" => Integer(p[:last_sessions]) } if p[:last_sessions]
+
+        {}
+      end
+
+      # The EvidenceCollector window discipline, applied to session ids: the
+      # agent's turns (by the task's command payload), distinct session ids,
+      # newest first, capped at the "last N conversations". Marker suppression
+      # unless `full` (a re-mine is explicit). A session's evidence ledger
+      #  is the grounding filter's input.
+      def resolve_sessions(agent, p, config)
+        ids = window_session_ids(agent, p, config)
+        ids = ids.reject { |sid| @harvest_store.mined?(sid) } unless EnvSchema.truthy?(p[:full])
+        ids = ids.first(Harvest::Miner::MAX_SESSIONS)
+
+        min_messages = (config["min_messages"] || DEFAULT_MIN_MESSAGES).to_i
+        idle_hours = (config["idle_hours"] || DEFAULT_IDLE_HOURS).to_i
+        ids.filter_map do |sid|
+          session = @session_store.find(sid)
+          next unless session
+
+          messages = session.messages.to_a
+          next if messages.size < min_messages # a 2-message session mines noise
+          # D10: the per-agent maturity bound is re-checked here — the engine
+          # scan's default is only the LOWER bound (a pack that wants 12 h is
+          # never mined at 6, and a manual CLI run over fresh traffic mines
+          # nothing).
+          next unless idle?(session.updated_at, idle_hours)
+
+          evidence = (session.evidence || {})["ids"] || []
+          { id: sid.to_s, messages: messages, evidence: Array(evidence).map(&:to_s) }
+        end
+      end
+
+      def window_session_ids(agent, p, config)
+        explicit = Array(p[:session_ids]).map(&:to_s)
+        return explicit unless explicit.empty?
+
+        since = AgentPayload.presence(p[:since])
+        tasks = @task_store.each_id
+                           .filter_map { |id| @task_store.find(id) }
+                           .select { |t| task_agent(t) == agent }
+                           .sort_by { |t| [t.created_at.to_s, t.id] }.reverse
+
+        tasks = tasks.select { |t| t.created_at.to_s >= since.to_s } if since
+
+        distinct = []
+        seen = {}
+        tasks.each do |t|
+          sid = presence(t.session_id)
+          next if sid.nil? || seen[sid]
+
+          seen[sid] = true
+          distinct << sid
+        end
+        return distinct if since
+
+        last = AgentPayload.presence(p[:last_sessions])
+        count = last ? Integer(last) : config_window(config)
+        distinct.first(count)
+      end
+
+      # The config's miner.window last_sessions, or the collector default.
+      def config_window(config)
+        configured = config.dig("miner", "window", "last_sessions")
+        configured ? Integer(configured) : DEFAULT_LAST_SESSIONS
+      end
+
+      def task_agent(task)
+        task.command.is_a?(Hash) ? task.command.dig("payload", "agent").to_s : ""
+      end
+
+      def idle?(updated_at, idle_hours)
+        return false if Coercion.blank?(updated_at)
+
+        Time.iso8601(updated_at.to_s) <= Time.now.utc - idle_hours * 3600
+      rescue ArgumentError
+        false
+      end
+
+      def presence(value)
+        Insika::Coercion.presence(value)
+      end
+
+      def max_proposals(p, config)
+        raw = p[:max_proposals] || config.dig("miner", "max_proposals")
+        raw ? Integer(raw) : DEFAULT_MAX_PROPOSALS
+      end
+
+      # The prompt: the transcript slices (masked through the   output
+      # filter — the   redaction rule), the evidence ids per
+      # session, the agent's CURRENT skill names (the model should not
+      # re-propose them), and the answer rules. The pack prompt replaces
+      # DEFAULT_PROMPT whole (the forge's half).
+      def build_prompt(config, sessions, agent)
+        base = Coercion.presence(config["prompt"]) || Harvest::DEFAULT_PROMPT
+        current_skills = current_skill_names(agent)
+        blocks = sessions.map do |s|
+          evidence = s[:evidence].join(", ")
+          lines = ["## Session #{s[:id]} (#{s[:messages].size} messages; " \
+                   "evidence ids: #{evidence.empty? ? '(none)' : evidence})"]
+          lines << render_transcript(s[:messages])
+          lines.join("\n")
+        end
+        <<~PROMPT
+          #{base.rstrip}
+
+          ## Conversations to mine
+
+          #{blocks.join("\n")}
+
+          ## Skills this agent already has (do not re-propose them)
+
+          #{current_skills.empty? ? "(none)" : current_skills.join(", ")}
+        PROMPT
+      end
+
+      def current_skill_names(agent)
+        return [] unless @skill_store
+
+        (@skill_store.names | @skill_store.names(agent: agent)).sort
+      end
+
+      def render_transcript(messages)
+        redacted, = Insika::Safety::Detectors.redact(
+          messages.each_with_index.map { |m, i| "[#{i}] #{m['role']}: #{m['content']}" }.join("\n")
+        )
+        redacted
+      end
+
+      # The safety order: negative list (D4) -> grounding (D3) -> dedup. A
+      # drop is COUNTED and NEVER written. `rejected` keys: the matched RULE
+      # ids for negative hits (each named — E2), "ungrounded"/"dedup" for the
+      # other two filters.
+      # -> [[raw skill], { key => count }]
+      def filter_skills(skills, sessions, agent, matcher)
+        rejected = Hash.new(0)
+        survivors = skills.select do |skill|
+          hits = negative_hits(skill)
+          if hits.any?
+            hits.each { |r| rejected[r.rule] += 1 }
+            false
+          elsif !grounded?(skill, sessions, matcher)
+            rejected["ungrounded"] += 1
+            false
+          elsif deduped?(skill, agent)
+            rejected["dedup"] += 1
+            false
+          else
+            true
+          end
+        end
+        [survivors, rejected.to_h]
+      end
+
+      # D4: every rejected-by-list candidate is logged with the matching rule
+      # id. The BODY is part of the match (the review fix): it is exactly what
+      # enters the model's context when the skill loads, so a banned phrase
+      # hidden there is the Hermes failure the design cites — the same text
+      # the grounding filter already reads (name+description+body). The name
+      # uses the stricter substring reading, the prose the word-boundary one.
+      def negative_hits(skill)
+        return [] unless @list
+
+        text = [skill["name"], skill["description"], skill["body"]].join(" ")
+        hits = @list.matches_name(skill["name"].to_s)
+        hits | @list.matches(text)
+      end
+
+      # D3: every reference must be in the union of the origin sessions'
+      # persisted evidence ids. The empty-ledger conservative reading (the
+      # enforcer's rule): a candidate whose origin sessions hold NO evidence
+      # ids and that carries a reference is dropped too. A skill with no
+      # references is not a grounding casualty. -> bool
+      def grounded?(skill, sessions, matcher)
+        text = [skill["name"], skill["description"], skill["body"]].join(" ")
+        refs = matcher.references(text)
+        return true if refs.empty?
+
+        union = sessions.flat_map { |s| s[:evidence] }.uniq
+        return false if union.empty?
+
+        missing = refs.reject { |r| union.include?(r) }
+        missing.empty?
+      end
+
+      # The dedup ledger: the same name already in the store's SkillStore
+      # (shared or agent scope) or an open (agent, name) tuple.
+      def deduped?(skill, agent)
+        name = skill["name"].to_s
+        store_has = @skill_store && (@skill_store.get(name) || @skill_store.get(name, agent: agent))
+        store_has || @harvest_store.open_pending?(agent: agent, name: name)
+      end
+
+      # The miner's schema drops + the filters' — one record; the keys never
+      # collide (the filters own rule ids and their own names).
+      def merge_rejected(dropped, filters)
+        dropped.merge(filters) { |_k, left, right| left + right }
+      end
+
+      def utility_model
+        return nil unless @settings_store
+
+        @settings_store.get["utility_model"]
+      end
+
+      def emit(type, **data)
+        @event_stream.emit(Insika::Event.new(
+                             type: type, data: data, meta: { at: Time.now.utc.iso8601 }
+                           ))
+      end
+    end
+  end
+end