insika 0.2.0 → 0.3.0

data/lib/insika/followup_store.rb

@@ -0,0 +1,306 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "time"
+
+module Insika
+  # the schedule records of the follow-up feature. The store owns
+  # the `pending|fired|cancelled|blocked` states (never written by a consumer —
+  # the task_store.rb state-machine idiom) and the (customer, reason) scans.
+  # It holds no policy (C2) and no contact cells (C3).
+  #
+  # States:
+  #   pending -> fired | cancelled | blocked
+  # `fired` is set ONLY inside the same transaction that created the synthetic
+  # task (D5 — the record and the turn commit together or together fail); a
+  # `blocked` record carries `blocked_reason` — auditable, never silent.
+  #
+  # Record key: "<tenant>:<agent>:<customer>:<at>:#{uuid}" — per-customer /
+  # per-agent scans are prefixes, and the scheduled `at` lives in the key so a
+  # fired record keeps its scheduled time for the A/B card. Blank tenant -> the
+  # literal "platform" (outcome_store.rb's rule — contact, follow-up and
+  # outcome keys share one tenant segment so the purge prefix scans line up).
+  class FollowupStore
+    SCOPE = "followups"
+    STATUSES = %w[pending fired cancelled blocked].freeze
+
+    Record = Data.define(:id, :tenant, :agent, :customer, :session_id, :at,
+                         :reason, :arm, :status, :task_id, :blocked_reason,
+                         :transport, :created_at, :updated_at, :fired_at)
+
+    def initialize(store:)
+      @store = store
+    end
+
+    # -> Record (status :pending). `at` must be a future ISO8601 (a Time or an
+    # ISO8601 string — ValidationError otherwise, "the follow-up would already
+    # be due"). The caller decided the arm (C1) and the transport (C7).
+    def create(tenant:, agent:, customer:, session_id:, at:, reason:, arm:,
+               transport: nil, id: SecureRandom.uuid, now: Time.now.utc)
+      time = parse_at(at)
+      if time <= now
+        raise Insika::ValidationError,
+              "follow-up #{id.inspect} would already be due (at #{time.iso8601} <= now)"
+      end
+      if (prior = pending_for(tenant: tenant, agent: agent, customer: customer, reason: reason))
+        raise Insika::ValidationError,
+              "a follow-up for (customer #{customer.inspect}, reason #{reason.inspect}) is already " \
+              "pending: #{prior.id}"
+      end
+
+      time = time.utc.iso8601
+      record = { "id" => id.to_s, "tenant" => tenant_id(tenant), "agent" => agent.to_s,
+                 "customer" => customer.to_s, "session_id" => session_id.to_s,
+                 "at" => time, "reason" => reason.to_s, "arm" => arm.to_s,
+                 "status" => "pending", "task_id" => nil, "blocked_reason" => nil,
+                 "transport" => transport.to_s, "created_at" => now.iso8601,
+                 "updated_at" => now.iso8601, "fired_at" => nil }
+      @store.set(SCOPE, key_for(record), record)
+      to_record(record)
+    end
+
+    # -> Record; NotFoundError on a nonexistent id. The only path out of
+    # pending besides the engine's fired/blocked. Idempotent: an
+    # already-cancelled record returns as-is (a repeat of the same call is not
+    # an error).
+    def cancel(id:, now: Time.now.utc)
+      mutate(id, now) do |record|
+        status = record["status"]
+        unless %w[pending cancelled].include?(status)
+          raise ArgumentError, "follow-up #{id}: cannot cancel a #{status} record"
+        end
+
+        record["status"] = "cancelled"
+      end
+    end
+
+    # The engine's atomic claim: pending -> fired, WITH task_id. Read-check-
+    # write inside Store#transaction (D5 — the record and the task commit
+    # together). A second claim raises. `fired_at` stamps WHEN the fire
+    # happened — the frequency ceiling and the A/B card count FIRES, never the
+    # scheduled time (a record booked days ago and fired after a tick outage
+    # must still count against the cap).
+    def transition_fired(id:, task_id:, now: Time.now.utc)
+      mutate(id, now) do |record|
+        raise ArgumentError, "follow-up #{id}: not pending (#{record['status']}) — it fires once" unless record["status"] == "pending"
+
+        record["status"] = "fired"
+        record["task_id"] = task_id.to_s
+        record["fired_at"] = now.utc.iso8601
+      end
+    end
+
+    # pending -> blocked, with the failing rule name. -> Record
+    def block(id:, reason:, now: Time.now.utc)
+      mutate(id, now) do |record|
+        raise ArgumentError, "follow-up #{id}: not pending (#{record['status']}) — only a pending record blocks" unless record["status"] == "pending"
+
+        record["status"] = "blocked"
+        record["blocked_reason"] = reason.to_s
+      end
+    end
+
+    # -> Record | nil
+    def find(id)
+      record = @store.get(SCOPE, key_for_id(id))
+      record && to_record(record)
+    end
+
+    # pending AND at <= now, oldest first (at, then id — determinism).
+    def due(now: Time.now.utc)
+      cutoff = now.iso8601
+      @store.list(SCOPE).filter_map do |k|
+        record = @store.get(SCOPE, k)
+        next unless record && record["status"] == "pending" && record["at"].to_s <= cutoff
+
+        to_record(record)
+      end.sort_by { |r| [r.at.to_s, r.id] }
+    end
+
+    # ALL records of one (tenant, agent) — the Follow-ups page's read (C10),
+    # lexicographic (the scheduled at first, so a fired record keeps its
+    # position for the A/B card).
+    def for_agent(tenant:, agent:)
+      prefix = "#{tenant_id(tenant)}:#{agent}:"
+      @store.list(SCOPE).filter_map do |k|
+        next unless k.start_with?(prefix)
+
+        to_record(@store.get(SCOPE, k))
+      end
+    end
+
+    # The dedup scans (D7): the OLDEST pending record of the pair (nil when
+    # none) and how many fired records of the customer fall inside the window
+    # (the frequency gate). `pending_for` scans the pair's keys in key order —
+    # the key embeds the scheduled `at` and the id, so the first pending record
+    # found IS the oldest pending.
+    def pending_for(tenant:, agent:, customer:, reason:)
+      pending_record_for(tenant: tenant, agent: agent, customer: customer, reason: reason)
+    end
+
+    def pending_for?(tenant:, agent:, customer:, reason:)
+      !pending_record_for(tenant: tenant, agent: agent, customer: customer, reason: reason).nil?
+    end
+
+    def fired_in_window(tenant:, customer:, since:)
+      boundary = since.iso8601
+      count = 0
+      @store.list(SCOPE).each do |k|
+        record = @store.get(SCOPE, k)
+        next unless record
+        next unless record["status"] == "fired"
+        next unless record["tenant"] == tenant_id(tenant)
+        next unless record["customer"] == customer.to_s
+        # the frequency gate counts FIRES, never the scheduled time — a record
+        # booked long ago and fired after a backlog still lands in the window.
+        next unless record["fired_at"].to_s >= boundary
+
+        count += 1
+      end
+      count
+    end
+
+    # Purge/prune (C11 — the LGPD footprint): one customer's records; a
+    # tenant's; records older than the cutoff — TERMINAL (fired/cancelled/
+    # blocked) OR pendings past their at (a zombie that will never fire;
+    # retention ages it out rather than firing late). All count-returning and
+    # nil-safe.
+
+    # cancels every PENDING record of the customer inside ONE
+    # transaction — the opt-out discipline (D2: a half-cancelled opt-out is
+    # the spam bug). Blocked/fired records are never touched. -> count
+    # cancelled.
+    def cancel_pending_for(tenant:, customer:)
+      count = 0
+      @store.transaction do
+        @store.list(SCOPE).each do |k|
+          record = @store.get(SCOPE, k)
+          next unless record && record["status"] == "pending"
+          next unless record["tenant"] == tenant_id(tenant)
+          next unless record["customer"] == customer.to_s
+
+          record["status"] = "cancelled"
+          record["updated_at"] = Time.now.utc.iso8601
+          @store.set(SCOPE, k, record)
+          count += 1
+        end
+      end
+      count
+    end
+
+    def purge_customer(tenant:, customer:)
+      removed = 0
+      @store.list(SCOPE).each do |k|
+        record = @store.get(SCOPE, k)
+        next unless record && record["tenant"] == tenant_id(tenant)
+        next unless record["customer"] == customer.to_s
+
+        @store.delete(SCOPE, k)
+        removed += 1
+      end
+      removed
+    end
+
+    def purge(tenant:)
+      prefix = "#{tenant_id(tenant)}:"
+      keys = @store.list(SCOPE).select { |k| k.start_with?(prefix) }
+      keys.each { |k| @store.delete(SCOPE, k) }
+      keys.size
+    end
+
+    # -> count removed.
+    def delete_older_than(time)
+      cutoff = time.utc.iso8601
+      removed = 0
+      @store.list(SCOPE).each do |k|
+        record = @store.get(SCOPE, k)
+        next unless record
+
+        terminal = record["status"] != "pending"
+        # TERMINAL records age by their updated_at; a PENDING record is a
+        # zombie when its scheduled `at` has passed (it will never fire —
+        # retention ages it out rather than firing late).
+        next unless (terminal && record["updated_at"].to_s < cutoff) ||
+                    (!terminal && record["at"].to_s < cutoff)
+
+        @store.delete(SCOPE, k)
+        removed += 1
+      end
+      removed
+    end
+
+    private
+
+    # -> Record | nil — the OLDEST pending record of the (agent, customer,
+    # reason) pair, if any (the dedup rule holds at creation AND at fire: the
+    # firer only fires the oldest pending per pair).
+    def pending_record_for(tenant:, agent:, customer:, reason:)
+      prefix = "#{tenant_id(tenant)}:#{agent}:"
+      @store.list(SCOPE).each do |k|
+        next unless k.start_with?(prefix)
+
+        record = @store.get(SCOPE, k)
+        next unless record && record["status"] == "pending"
+        next unless record["customer"] == customer.to_s
+        next unless record["reason"] == reason.to_s
+
+        return to_record(record)
+      end
+      nil
+    end
+
+    def mutate(id, now)
+      @store.transaction do
+        record = @store.get(SCOPE, key_for_id(id))
+        raise Insika::NotFoundError, "follow-up not found: #{id}" if record.nil?
+
+        yield record
+        record["updated_at"] = now.utc.iso8601
+        @store.set(SCOPE, key_for_id(id), record)
+        to_record(record)
+      end
+    end
+
+    def key_for(record)
+      # tenant + agent + customer + the scheduled at + id: per-customer /
+      # per-agent scans are prefixes, the at-first ordering keeps a fired
+      # record at its scheduled position (the A/B card's lexicographic list).
+      "#{record['tenant']}:#{record['agent']}:#{record['customer']}:#{record['at']}:#{record['id']}"
+    end
+
+    def key_for_id(id)
+      # the id is the LAST segment, so a find is a suffix match over the
+      # scope's keys (there is no stable prefix for an id alone).
+      @store.list(SCOPE).find { |k| k.end_with?(":#{id}") }
+    end
+
+    def parse_at(at)
+      case at
+      when Time then at.utc
+      else
+        begin
+          Time.iso8601(at.to_s).utc
+        rescue ArgumentError
+          raise Insika::ValidationError, "follow-up `at` must be an ISO8601 time, got: #{at.inspect}"
+        end
+      end
+    end
+
+    def tenant_id(tenant)
+      t = tenant.to_s
+      t.empty? ? "platform" : t
+    end
+
+    def to_record(rec)
+      return nil if rec.nil?
+
+      Record.new(id: rec["id"], tenant: rec["tenant"], agent: rec["agent"],
+                 customer: rec["customer"], session_id: rec["session_id"],
+                 at: rec["at"], reason: rec["reason"], arm: rec["arm"],
+                 status: rec["status"], task_id: rec["task_id"],
+                 blocked_reason: rec["blocked_reason"], transport: rec["transport"],
+                 created_at: rec["created_at"], updated_at: rec["updated_at"],
+                 fired_at: rec["fired_at"])
+    end
+  end
+end

data/lib/insika/funnel_declaration.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Insika
+  # the parsed funnel declaration of ONE agent — the ONLY shape
+  # the engine accepts, shared by the fold, the doctor, the Studio and the
+  # freeze command. Pure value object: it never touches a store, and the engine
+  # never hard-codes a stage name (D1) — `stages`/`primary`/`advance_on` are the
+  # forge's vocabulary, carried as data.
+  #
+  # `parse` returns nil on a malformed hash (D8: the fold skips that agent, the
+  # doctor explains the exact defect); `parse!` raises Insika::ValidationError
+  # naming the field. `attribution_window` is carried data, never computed (D4).
+  class FunnelDeclaration
+    WINDOW_RE = /\A\d+h\z/
+
+    attr_reader :stages, :advance_on, :primary, :attribution_window
+
+    def self.parse(hash)
+      new(hash)
+    rescue Insika::ValidationError
+      nil
+    end
+
+    def self.parse!(hash)
+      new(hash)
+    end
+
+    def initialize(hash)
+      raise Insika::ValidationError, "funnel: declaration must be a Hash" unless hash.is_a?(Hash)
+
+      h = hash.transform_keys(&:to_s)
+      @stages = stages_of(h)
+      @advance_on = advance_on_of(h, @stages)
+      @primary = primary_of(h, @stages)
+      @attribution_window = window_of(h)
+      freeze
+    end
+
+    # position of a stage in the declared order. -> Integer | nil
+    def index_of(stage)
+      index = @stages.index(stage.to_s)
+      index.nil? ? nil : index
+    end
+
+    # the first declared stage — the funnel's denominator.
+    def first_stage = @stages.first
+
+    # "72h" -> 72 (Integer). Always valid once parsed.
+    def window_hours = @attribution_window.to_i
+
+    # Value-object equality: two declarations parsed from equal input (symbol
+    # or string keys) are the same declaration.
+    def ==(other)
+      other.is_a?(FunnelDeclaration) &&
+        @stages == other.stages && @advance_on == other.advance_on &&
+        @primary == other.primary && @attribution_window == other.attribution_window
+    end
+    alias eql? ==
+
+    def hash
+      [@stages, @advance_on, @primary, @attribution_window].hash
+    end
+
+    private
+
+    def stages_of(hash)
+      list = hash["stages"]
+      raise Insika::ValidationError, "funnel.stages: must be a non-empty Array of non-blank Strings" unless valid_stages?(list)
+
+      list
+    end
+
+    def valid_stages?(list)
+      list.is_a?(Array) && !list.empty? && list.all? { |s| s.is_a?(String) && !s.strip.empty? } &&
+        list.uniq.length == list.length
+    end
+
+    def advance_on_of(hash, stages)
+      map = hash["advance_on"]
+      raise Insika::ValidationError, "funnel.advance_on: must be a non-empty Hash of kind => stage" unless map.is_a?(Hash) && !map.empty?
+
+      map = map.transform_keys(&:to_s)
+      unless map.all? { |kind, stage| !kind.strip.empty? && stage.is_a?(String) && stages.include?(stage) }
+        raise Insika::ValidationError, "funnel.advance_on: every value must be one of the declared stages"
+      end
+
+      map
+    end
+
+    def primary_of(hash, stages)
+      primary = hash["primary"]
+      raise Insika::ValidationError, "funnel.primary: must be one of the declared stages" unless primary.is_a?(String) && stages.include?(primary)
+
+      primary
+    end
+
+    def window_of(hash)
+      window = hash["attribution_window"]
+      unless window.is_a?(String) && WINDOW_RE.match?(window)
+        raise Insika::ValidationError, "funnel.attribution_window: must match /\\A\\d+h\\z/ (e.g. \"72h\")"
+      end
+
+      window
+    end
+  end
+end

data/lib/insika/funnel_fold.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require "time"
+require "securerandom"
+
+module Insika
+  # the tick-driven fold of WS7 outcome records into the
+  # pack-declared stages (the tick's cadence). One pass per claim
+  # window; the fold is idempotent across crashes via the per-pair {at, ids}
+  # cursor (D3) and per-pair transactions. Pairs whose declaration is absent or
+  # malformed are skipped (D8), as are outcome kinds the declaration does not
+  # map — the funnel shows the hole. Nothing else: no attribution
+  # (D4), no stage vocabulary of its own (D1).
+  class FunnelFold
+    SCOPE = "funnel_fold"
+    KEY = "claim"
+    DEFAULT_WINDOW = 300 # seconds; one folding worker per window
+
+    def initialize(outcome_store:, funnel_store:, profiles:, store:,
+                   window: DEFAULT_WINDOW, now: nil)
+      @outcome_store = outcome_store
+      @funnel_store = funnel_store
+      @profiles = profiles
+      @store = store
+      @window = window
+      @now = now
+    end
+
+    # One pass. -> { claimed: true, folded: Integer, skipped: Integer, pairs: Integer }
+    #             | { claimed: false }  (another worker holds the window — the
+    #               retention.rb:74 shape, so the tick summary reads uniformly)
+    def run
+      return { claimed: false } unless claim
+
+      folded = 0
+      skipped = 0
+      pairs = 0
+      # The pairs come from the OUTCOME keys (prefix scan, no record reads);
+      # each pair's records are read by `for_pair`, so a pair without a
+      # declaration is never even read (a full `all` would read every record
+      # of every pair — the O(n) the key shape is there to avoid).
+      @outcome_store.pairs.each do |pair|
+        begin
+          declaration = declaration_for(pair[:agent])
+          next unless declaration
+
+          folded += fold_pair(tenant: pair[:tenant], agent: pair[:agent],
+                              declaration: declaration, skipped: -> { skipped += 1 })
+          pairs += 1
+        rescue StandardError
+          # a broken pair must not hold every other store's funnel hostage —
+          # its own transaction already rolled back; the pass keeps folding.
+          next
+        end
+      end
+      { claimed: true, folded: folded, skipped: skipped, pairs: pairs }
+    end
+
+    # From scratch (E1's "recompute matches the incremental fold", and the
+    # repair path for backfilled outcomes): wipes the pair's day cells and
+    # rebuilds them from every outcome record. `tenant:` takes the store's
+    # spellings alike (nil/""/"platform" = the no-tenant pair) — `for_pair` and
+    # `delete_days` normalize at their own key boundaries, so no tenant is
+    # ever mixed with another (a "platform" recompute never folds "acme"'s
+    # records). -> folded count.
+    def recompute(tenant:, agent:, declaration:)
+      @funnel_store.delete_days(tenant: tenant, agent: agent)
+      fold_pair(tenant: tenant, agent: agent, declaration: declaration, full: true)
+    end
+
+    private
+
+    # -> FunnelDeclaration | nil — absent or malformed declarations read as
+    # "nothing to fold" (D8); never raises.
+    def declaration_for(agent)
+      profile = @profiles.fetch(agent)
+      return nil unless profile
+
+      Insika::FunnelDeclaration.parse(profile.funnel)
+    end
+
+    # The fold for one pair: cursor-filtered, ONE transaction (all adds + the
+    # cursor write — D3: a crash leaves either the old or the new state, never
+    # a half-fold). `full: true` (recompute) ignores the cursor — every record
+    # folds and the cursor rewrites from the newest. Records are read by key
+    # prefix, optionally skipping the keys older than the cursor's day without
+    # reading them. -> folded count.
+    def fold_pair(tenant:, agent:, declaration:, records: nil, skipped: nil, full: false)
+      cursor = @funnel_store.cursor(tenant: tenant, agent: agent)
+      # The cursor's DAY is the prefix bound: records on that day may still be
+      # newer than the cursor (boundary handling below); anything older is
+      # already folded. `full` reads the whole pair.
+      records ||= @outcome_store.for_pair(tenant: tenant, agent: agent,
+                                          since_date: full ? nil : cursor["at"]&.[](0, 10))
+      new_records = records.select { |r| full || new_record?(r, cursor) }
+                          .sort_by { |r| [r.at.to_s, r.id] }
+      return 0 if new_records.empty? && !full
+      if new_records.empty?
+        # recompute of an emptied pair: nothing to fold, but the stale cursor
+        # must not outlive its records (a later backfill would be invisible).
+        @funnel_store.set_cursor(tenant: tenant, agent: agent, at: nil, ids: [])
+        return 0
+      end
+
+      folded = 0
+      # Grouped by DAY (the cell key), never by the full timestamp: N events on
+      # one day must be ONE read-modify-write of the day cell, not N.
+      counts_by_day = Hash.new { |h, k| h[k] = Hash.new(0) }
+      at_by_day = {}
+      newest_at = full ? nil : cursor["at"]
+      ids_at_newest = []
+
+      new_records.each do |record|
+        # The cursor advances over SKIPPED records too (they are not new any
+        # more): a pair whose integration only emits unmapped kinds would
+        # otherwise re-read the same records every window forever.
+        if newest_at.nil? || record.at.to_s > newest_at.to_s
+          newest_at = record.at.to_s
+          ids_at_newest = [record.id]
+        elsif record.at.to_s == newest_at.to_s
+          ids_at_newest << record.id
+        end
+
+        index = declaration.index_of(declaration.advance_on[record.outcome])
+        if index.nil?
+          skipped&.call
+          next
+        end
+
+        day = record.at.to_s[0, 10] # the record's at is always UTC ISO8601
+        declaration.stages[0..index].each { |stage| counts_by_day[day][stage] += 1 }
+        at_by_day[day] ||= record.at.to_s
+        folded += 1
+      end
+
+      @store.transaction do
+        counts_by_day.each do |day, counts|
+          @funnel_store.add(tenant: tenant, agent: agent, at: at_by_day[day], counts: counts)
+        end
+        @funnel_store.set_cursor(tenant: tenant, agent: agent,
+                                 at: newest_at, ids: ids_at_newest)
+      end
+      folded
+    end
+
+    # `at > cursor.at` or boundary-equal and not already folded (D3). The at
+    # field is ISO8601 UTC, so the comparison is lexicographic; a record with
+    # `at` older than the cursor (a backfill) is missed incrementally —
+    # `recompute` is the repair path.
+    def new_record?(record, cursor)
+      at = cursor["at"]
+      return true if at.nil?
+      return true if record.at.to_s > at.to_s
+
+      record.at.to_s == at.to_s && !cursor["ids"].include?(record.id)
+    end
+
+    # The claim window (tick.rb:93's idiom — read-check-write on one key inside
+    # a transaction): the scan is O(outcome records), fine at 288 passes/day,
+    # but it does not ride every 60 s tick.
+    def claim
+      now_time = @now || Time.now.utc
+      @store.transaction do
+        current = @store.get(SCOPE, KEY)
+        last = current && begin
+          Time.iso8601(current["claimed_at"].to_s)
+        rescue ArgumentError
+          nil # a corrupted claim is not a claim — take the window
+        end
+        if last.nil? || (now_time - last) >= @window
+          @store.set(SCOPE, KEY, { "claimed_at" => now_time.iso8601 })
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end