insika 0.2.0 → 0.7.0

data/lib/insika/safety/output_validator.rb

@@ -27,13 +27,19 @@ module Insika
     class OutputValidator
       # `ask_factory` (optional): ->(config) { ->(prompt){text} | nil }, built by the
       # Safety::Factory from the utility_model. nil = deterministic only.
-      def initialize(ask_factory: nil)
+      # `grounding` (optional): a GroundingValidator — its step runs
+      # FIRST in #call, before the `config.output` gate (D9: grounding is
+      # evidence integrity, independent of the guardrails opt-in).
+      def initialize(ask_factory: nil, grounding: nil)
         @ask_factory = ask_factory
+        @grounding = grounding
       end
 
       # after_task hook body. Idempotent and defensive: never raises out (a hook
-      # error must not fail a committed turn).
+      # error must not fail a committed turn). Grounding runs BEFORE the output
+      # gate so an agent with guardrails off and grounding on still gets the check.
       def call(state)
+        state = @grounding&.call(state) || state
         config = Config.from_profile(state.profile)
         return state unless config.output
 
@@ -41,7 +47,7 @@ module Insika
         return state if text.empty?
 
         flags = []
-        flags.concat(deterministic_flags(text))
+        flags.concat(deterministic_flags(text, config.corpus))
         flags.concat(llm_flags(text, config)) if config.moderator?
 
         state.guardrail_flags = Array(state.guardrail_flags) + flags unless flags.empty?
@@ -55,8 +61,8 @@ module Insika
       # Residual PII/secret that somehow reached the final text (the stream filter
       # should have caught it — this is defense in depth, and the flag itself
       # carries category counts, never the raw value).
-      def deterministic_flags(text)
-        _redacted, counts = Detectors.redact(text)
+      def deterministic_flags(text, corpus)
+        _redacted, counts = Detectors.redact(text, corpus: corpus)
         return [] if counts.empty?
 
         [{ category: "pii_residual", source: "deterministic", detail: counts.map { |k, v| "#{k}:#{v}" }.join(",") }]

data/lib/insika/schedule.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+module Insika
+  # the parsed recurring-schedule declaration of ONE agent —
+  # the ONLY shape the engine accepts, shared by the ScheduleEngine, the
+  # doctor and the Studio. Pure value object (the followup-policy precedent).
+  #
+  # One schedule: a name, a trigger (`cron` OR `every`, never both), a
+  # timezone for cron materialization, the synthetic inbound message, a
+  # session mode (a fresh session per run, or a standing one), per-run
+  # overrides (turn_timeout / max_tool_calls / model) and an enabled flag.
+  #
+  # `parse` returns nil on a malformed hash (the engine SKIPS, the doctor
+  # explains, the Studio refuses); `parse!` raises Insika::ValidationError
+  # naming the exact defect.
+  class Schedule
+    SESSION_MODES = %w[new fixed].freeze
+    OVERRIDE_KEYS = %w[turn_timeout max_tool_calls model].freeze
+    ID_RE = /\A[a-z][a-z0-9_-]*\z/
+
+    attr_reader :id, :every, :cron, :tz, :message, :session_mode, :session_id,
+                :overrides, :enabled
+
+    def self.parse(hash)
+      new(hash)
+    rescue Insika::ValidationError
+      nil
+    end
+
+    def self.parse!(hash)
+      new(hash)
+    end
+
+    def initialize(hash)
+      raise Insika::ValidationError, "schedule: declaration must be a Hash" unless hash.is_a?(Hash)
+
+      h = hash.transform_keys(&:to_s)
+      @id = id_of(h)
+      @every = every_of(h)
+      @cron = cron_of(h)
+      if @cron.nil? && @every.nil?
+        raise Insika::ValidationError,
+              "schedule '#{@id}': a trigger is required — declare cron or every"
+      end
+      @tz = tz_of(h)
+      @message = message_of(h)
+      @session_mode = session_mode_of(h)
+      @session_id = session_id_of(h)
+      @overrides = overrides_of(h)
+      @enabled = h.key?("enabled") ? h["enabled"] == true : true
+      freeze
+    end
+
+    def cron? = !@cron.nil?
+    def every? = !@every.nil?
+    def fixed_session? = @session_mode == "fixed"
+
+    def to_h
+      { "id" => @id, "cron" => @cron, "every" => @every, "tz" => @tz,
+        "message" => @message, "session_mode" => @session_mode,
+        "session_id" => @session_id, "overrides" => @overrides,
+        "enabled" => @enabled }.compact
+    end
+
+    private
+
+    def id_of(h)
+      id = h["id"].to_s.strip.downcase
+      unless ID_RE.match?(id)
+        raise Insika::ValidationError,
+              "schedule.id must match #{ID_RE.inspect}, got: #{h['id'].inspect}"
+      end
+
+      id
+    end
+
+    def every_of(h)
+      every = h["every"]
+      return nil if every.nil?
+
+      unless every.is_a?(Integer) && every.positive?
+        raise Insika::ValidationError,
+              "schedule.every must be a positive integer of seconds, got: #{h['every'].inspect}"
+      end
+
+      every
+    end
+
+    def cron_of(h)
+      cron = h["cron"]
+      return nil if cron.nil?
+
+      if h.key?("every") && !h["every"].nil?
+        raise Insika::ValidationError,
+              "schedule '#{@id}': cron and every are mutually exclusive — declare exactly one trigger"
+      end
+
+      Insika::Cron.new(cron) # raises ValidationError on a malformed expression
+      cron.to_s
+    end
+
+    def tz_of(h)
+      tz = h["tz"].to_s
+      tz = "Etc/UTC" if tz.empty?
+      unless Insika::Timezone.known?(tz)
+        raise Insika::ValidationError, "schedule '#{@id}'.tz is not a valid IANA timezone: #{tz.inspect}"
+      end
+
+      tz
+    end
+
+    def message_of(h)
+      message = h["message"]
+      if Coercion.blank?(message)
+        raise Insika::ValidationError,
+              "schedule '#{@id}'.message is required — the synthetic inbound that kicks each run"
+      end
+
+      message.to_s
+    end
+
+    def session_mode_of(h)
+      mode = h["session_mode"]
+      return "new" if mode.nil?
+
+      mode = mode.to_s
+      unless SESSION_MODES.include?(mode)
+        raise Insika::ValidationError,
+              "schedule '#{@id}'.session_mode must be one of #{SESSION_MODES.inspect}, got: #{h['session_mode'].inspect}"
+      end
+
+      mode
+    end
+
+    def session_id_of(h)
+      session_id = h["session_id"]
+      return nil if session_id.nil?
+
+      session_id.to_s
+    end
+
+    def overrides_of(h)
+      overrides = h["overrides"]
+      return nil if overrides.nil?
+
+      raise Insika::ValidationError, "schedule '#{@id}'.overrides must be a Hash" unless overrides.is_a?(Hash)
+
+      overrides = overrides.transform_keys(&:to_s)
+      unknown = overrides.keys - OVERRIDE_KEYS
+      unless unknown.empty?
+        raise Insika::ValidationError,
+              "schedule '#{@id}'.overrides: unknown key(s) #{unknown.inspect} " \
+              "(allowed: #{OVERRIDE_KEYS.join(', ')})"
+      end
+
+      overrides.each do |key, value|
+        # model is a provider/model REF — validated at declaration, not left
+        # for model-resolution time; everything else is an integer ceiling.
+        if key == "model"
+          if !value.is_a?(String) || value.strip.empty?
+            raise Insika::ValidationError,
+                  "schedule '#{@id}'.overrides.model must be a non-blank String (a " \
+                  "model ref), got: #{value.inspect}"
+          end
+          next
+        end
+
+        unless value.is_a?(Integer) && value.positive?
+          raise Insika::ValidationError,
+                "schedule '#{@id}'.overrides.#{key} must be a positive Integer, got: #{value.inspect}"
+        end
+      end
+
+      overrides
+    end
+  end
+end

data/lib/insika/schedule_engine.rb

@@ -0,0 +1,314 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  # the tick-driven FIRER of recurring schedules: the engine's
+  # fourth duty (after the outbox drain, retention/funnel and the follow-up
+  # firer). One pass per claim window (the followup/funnel idiom); each due
+  #   schedule is claimed transactionally — re-read inside its own transaction,
+  #   so two workers racing a window serialize on the backend lock and exactly
+  #   one fires (the multi-worker at-most-once claim, per row).
+  #
+  # Gating order:
+  #
+  #   declared? (reconciled from the profiles) -> enabled + due -> no-catch-up
+  #   (a window older than one claim window is MISSED, recorded, never
+  #   replayed) -> overlap (the last task still live) -> budget (a hard
+  #   window at/over cap) -> FIRE.
+  #
+  # Skips are DATA, never silent: `last_skip { at, reason }` on the row, for
+  # the Studio. The engine never queues — a skipped window advances the
+  # schedule's lattice. The turn it creates is delivered by the existing
+  # pipeline (it holds no channel code, like the FollowupEngine).
+  class ScheduleEngine
+    SCOPE = "schedule_fire"
+    KEY = "claim"
+    DEFAULT_WINDOW = 300 # seconds; one firing worker per window
+    TENANT = "platform"  # the single-tenant default (ledger rules) — see `tenant_for`
+
+    ACTIVE_STATUSES = %i[queued running waiting paused].freeze
+
+    # The tenant a SCHEDULED turn declares. Every other turn gets its tenant
+    # from its CALLER (an authenticated tenant token, a Command built with
+    # `tenant:`) — a scheduled turn has no caller, so it is the agent's OWN
+    # declaration instead: `metadata["tenant"]`, the same "stable per agent,
+    # from the pack" home `store_id` already lives in (never the model, never
+    # a policy — just a fact the pack states). Absent -> the single-tenant
+    # default, unchanged for every profile that does not declare one.
+    #
+    # PUBLIC and STATELESS on purpose: the Studio's own schedule list
+    # (`studio/app.rb`) must resolve to the exact SAME tenant the fire path
+    # uses, or a declared schedule becomes invisible there — one formula, two
+    # callers, never a second copy to drift.
+    def self.tenant_for(profile)
+      meta = profile.respond_to?(:metadata) ? profile.metadata : nil
+      Insika::Coercion.presence(meta && meta["tenant"]) || TENANT
+    end
+
+    def initialize(store:, schedule_store:, task_store:, session_store:,
+                   profiles:, executor:, budget_ledger: nil, event_stream: nil,
+                   window: DEFAULT_WINDOW, now: nil)
+      @store = store
+      @schedule_store = schedule_store
+      @task_store = task_store
+      @session_store = session_store
+      @profiles = profiles
+      @executor = executor
+      @budget_ledger = budget_ledger
+      @event_stream = event_stream
+      @window = window
+      @now = now
+    end
+
+    # -> { claimed: false }
+    #  | { claimed: true, fired: N, skipped: N, errors: N,
+    #      skip_reasons: { "reason" => N } }
+    # A StoreError on ONE schedule aborts THAT schedule's transaction
+    # (rescued, counted, the loop continues) — a broken row must not hold the
+    # other schedules' runs hostage.
+    def run
+      now_time = @now || Time.now.utc
+      return { claimed: false } unless claim_window(now_time)
+
+      sync_from_profiles(now_time)
+
+      fired = 0
+      skipped = 0
+      errors = 0
+      reasons = Hash.new(0)
+
+      @schedule_store.due(now: now_time).each do |record|
+        begin
+          outcome = fire_record(record, now_time)
+          case outcome
+          when :fired then fired += 1
+          when Array
+            # a skip is recorded on the row (never silent) and counted here.
+            skipped += 1
+            reasons[outcome[1].to_s] += 1
+          end
+        rescue StandardError
+          # a broken schedule must not hold the other schedules' runs
+          # hostage — its own transaction already rolled back.
+          errors += 1
+        end
+      end
+
+      { claimed: true, fired: fired, skipped: skipped, errors: errors,
+        skip_reasons: reasons }
+    end
+
+    private
+
+    # Reconciliation: the PROFILES are the source of the declarations (DSL /
+    # API / Studio); the store rows are the derived view the fire path reads.
+    # One pass: drop rows whose agent no longer exists, then upsert each
+    # profile's declared schedules (and drop each agent's undeclared rows).
+    def sync_from_profiles(now_time)
+      ids = @profiles.ids.map(&:to_s)
+      @store.transaction do
+        @schedule_store.all.each do |row|
+          @schedule_store.delete(tenant: row.tenant, agent: row.agent, id: row.id) unless ids.include?(row.agent)
+        end
+        @profiles.all.each do |profile|
+          schedules = profile.respond_to?(:schedules) ? profile.schedules : nil
+          @schedule_store.sync_declared(tenant: self.class.tenant_for(profile), agent: profile.id,
+                                        schedules: schedules, now: now_time)
+        end
+      end
+    end
+
+    # -> :fired | [:skipped, reason] — claimed per row, inside ONE
+    # transaction: the re-read, the gates, the task creation and the lattice
+    # advance commit together or not at all — and the SPAWN happens AFTER the
+    # commit, so a spawn failure never unwinds the fire (the durable :queued
+    # task is the recovery sweep's to handle). `next`, never `return`, inside
+    # the block (a non-local return skips the backend's COMMIT).
+    def fire_record(record, now_time)
+      task = nil
+      outcome = nil
+      spawn_profile = nil
+      @store.transaction do
+        current = @schedule_store.find(tenant: record.tenant, agent: record.agent, id: record.id)
+        # a racing worker already advanced/deleted the row — nothing for this pass.
+        unless current
+          outcome = [:skipped, :stale]
+          next
+        end
+        unless current.enabled
+          outcome = [:skipped, :stale]
+          next
+        end
+        next_at = Time.iso8601(current.next_fire_at.to_s)
+        unless next_at <= now_time
+          outcome = [:skipped, :stale] # the racing worker claimed the window
+          next
+        end
+
+        # the no-catch-up policy: a window older than ONE claim window is
+        # MISSED, not replayed — the lattice advances and the row records it.
+        if next_at < now_time - @window
+          @schedule_store.mark_skip(id: current.id, tenant: current.tenant,
+                                    agent: current.agent, reason: :late,
+                                    next_fire_at: next_after(current, now_time),
+                                    now: now_time)
+          outcome = [:skipped, :late]
+          next
+        end
+
+        # overlap: the previous run is still live — skip + record, never a queue.
+        if overlap?(current)
+          @schedule_store.mark_skip(id: current.id, tenant: current.tenant,
+                                    agent: current.agent, reason: :overlap,
+                                    next_fire_at: next_after(current, now_time),
+                                    now: now_time)
+          outcome = [:skipped, :overlap]
+          next
+        end
+
+        profile = @profiles.fetch(current.agent)
+        if profile && budget_exhausted?(profile, current, now_time)
+          @schedule_store.mark_skip(id: current.id, tenant: current.tenant,
+                                    agent: current.agent, reason: :budget,
+                                    next_fire_at: next_after(current, now_time),
+                                    now: now_time)
+          outcome = [:skipped, :budget]
+          next
+        end
+
+        task = commit_run(current, profile, now_time)
+        spawn_profile = derived_profile(current, profile)
+        outcome = :fired
+      end
+      return outcome || [:skipped, :stale] unless task
+
+      # AFTER the commit: the spawn. A failure propagates to the pass (counted
+      # as an error) — the fire already committed, and the :queued task is
+      # recovered by the tick's sweep.
+      @executor.spawn_in_session(task, profile: spawn_profile)
+      emit_fired(record, task.id)
+      :fired
+    end
+
+    # The atomic claim inside the pass's transaction: the task and the
+    # schedule's state (last run, task id, next fire) commit together or not
+    # at all (the follow-up firer's D5 shape).
+    def commit_run(current, profile, now_time)
+      session_id = resolve_session(current)
+      command = {
+        "type" => "scheduled_run",
+        "session_id" => session_id,
+        "payload" => {
+          "agent" => current.agent, "session_id" => session_id,
+          "message" => current.message, "origin" => Insika::MessageOrigin::SCHEDULED,
+          "schedule_id" => current.id
+        },
+        "meta" => { "tenant" => current.tenant, "transport" => "schedule" }
+      }
+      task = @task_store.create(command: command, session_id: session_id)
+      @schedule_store.transition_fire(id: current.id, tenant: current.tenant,
+                                      agent: current.agent, task_id: task.id,
+                                      next_fire_at: next_after(current, now_time),
+                                      now: now_time)
+      task
+    end
+
+    # -> session_id for the run. session_mode "new" = a fresh session per run
+    # (the report case); "fixed" = the declared session, created on first run
+    # (the "standing assistant" case).
+    def resolve_session(current)
+      if current.session_mode == "fixed"
+        sid = current.session_id.to_s
+        sid = "sched-#{current.agent}-#{current.id}" if sid.empty?
+        @session_store.create(id: sid) unless @session_store.find(sid)
+        sid
+      else
+        @session_store.create.id
+      end
+    end
+
+    # The profile the turn runs on: the base profile with the schedule's
+    # overrides merged (per-schedule ceiling, never a store-wide change). The
+    # base is untouched — a second schedule cannot see a sibling's overrides.
+    def derived_profile(current, base)
+      overrides = current.overrides
+      return base if overrides.nil? || overrides.empty?
+
+      limits = base.limits.dup
+      limits[:turn_timeout] = overrides["turn_timeout"] if overrides["turn_timeout"]
+      limits[:max_tool_calls] = overrides["max_tool_calls"] if overrides["max_tool_calls"]
+      Insika::AgentProfile.build(**base.to_h.merge(limits: limits,
+                                                   model: overrides["model"] || base.model))
+    end
+
+    # The next lattice point after `now`: for `every`, the next interval
+    # boundary; for cron, the next expression occurrence in the schedule's tz.
+    # nil (a cron that can never fire) makes the row never due again.
+    def next_after(current, now_time)
+      if current.every
+        base = Time.iso8601(current.next_fire_at.to_s)
+        base + ((now_time - base).to_i / current.every + 1) * current.every
+      else
+        Insika::Cron.new(current.cron).next_after(now_time, tz: current.tz)
+      end
+    end
+
+    def overlap?(current)
+      task_id = current.last_task_id.to_s
+      return false if task_id.empty?
+
+      task = @task_store.find(task_id)
+      task && ACTIVE_STATUSES.include?(task.status)
+    end
+
+    # A HARD budget at/over a window cap = skip (the edge would fail the turn
+    # anyway — this refuses to even queue it). A soft budget crosses and runs;
+    # the ledger warns as usual. Mirror of EdgeLimiter#budget_windows — the
+    # SOFT half is the edge's, this is the schedule gate's.
+    def budget_exhausted?(profile, current, now_time)
+      budget = profile.respond_to?(:budget) ? profile.budget : nil
+      return false if budget.nil? || @budget_ledger.nil?
+
+      soft = budget["soft"] == true
+      %i[daily monthly].any? do |window|
+        cap = budget[window.to_s].to_i
+        next false unless cap.positive?
+
+        spent = @budget_ledger.current(tenant: current.tenant, agent: current.agent,
+                                       now: now_time)[window]
+        !soft && spent >= cap
+      end
+    end
+
+    def emit_fired(record, task_id)
+      return unless @event_stream
+
+      @event_stream.emit(Insika::Event.new(
+                           type: :schedule_fired,
+                           data: { id: record.id, agent: record.agent, task_id: task_id },
+                           meta: { tenant: record.tenant, at: Time.now.utc.iso8601 }
+                         ))
+    end
+
+    # The claim window (the funnel_fold.rb idiom — read-check-write on one key
+    # inside a transaction): the O(n) scans never ride the 60 s tick, and two
+    # workers racing a pass serialize on the backend's lock.
+    def claim_window(now_time)
+      @store.transaction do
+        current = @store.get(SCOPE, KEY)
+        last = current && begin
+          Time.iso8601(current["claimed_at"].to_s)
+        rescue ArgumentError
+          nil # a corrupted claim is not a claim — take the window
+        end
+        if last.nil? || (now_time - last) >= @window
+          @store.set(SCOPE, KEY, { "claimed_at" => now_time.iso8601 })
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end