insika 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +145 -17
  3. data/README.md +7 -1
  4. data/bin/insika +211 -8
  5. data/docs/AGENTS.md +377 -2
  6. data/docs/API.md +56 -0
  7. data/docs/CHANNELS.md +99 -9
  8. data/docs/CONTEXT.md +127 -16
  9. data/docs/DEPLOY.md +5 -2
  10. data/docs/EVALS.md +19 -0
  11. data/docs/FACTS.md +135 -0
  12. data/docs/HARVEST.md +117 -0
  13. data/docs/LOADTEST.md +14 -8
  14. data/docs/OBSERVABILITY.md +54 -0
  15. data/docs/REFINEMENT.md +3 -3
  16. data/docs/RELEASING.md +27 -0
  17. data/docs/RUNNING-LOCAL.md +3 -3
  18. data/docs/SECURITY.md +61 -0
  19. data/docs/SKILLS.md +2 -2
  20. data/docs/SOAK.md +127 -0
  21. data/docs/TOOLS.md +70 -2
  22. data/docs/domain.md +115 -0
  23. data/docs/index.md +1 -1
  24. data/lib/insika/agent_profile.rb +145 -10
  25. data/lib/insika/balloon_splitter.rb +102 -0
  26. data/lib/insika/budget_ledger.rb +34 -3
  27. data/lib/insika/cache_series_store.rb +49 -0
  28. data/lib/insika/channel_delivery.rb +118 -13
  29. data/lib/insika/channels/relay.rb +77 -3
  30. data/lib/insika/channels/web.rb +2 -2
  31. data/lib/insika/chat_builder.rb +84 -1
  32. data/lib/insika/checkpoint_store.rb +16 -0
  33. data/lib/insika/commands/agent_payload.rb +3 -3
  34. data/lib/insika/commands/cancel_followup.rb +49 -0
  35. data/lib/insika/commands/delete_tenant_data.rb +95 -0
  36. data/lib/insika/commands/export_customer_memory.rb +48 -0
  37. data/lib/insika/commands/forget_customer.rb +117 -0
  38. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  39. data/lib/insika/commands/gate_harvest.rb +138 -0
  40. data/lib/insika/commands/gate_refinement.rb +1 -1
  41. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  42. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  43. data/lib/insika/commands/memory_put_fact.rb +23 -4
  44. data/lib/insika/commands/promote_harvest.rb +130 -0
  45. data/lib/insika/commands/record_outcome.rb +46 -0
  46. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  47. data/lib/insika/commands/reject_harvest.rb +38 -0
  48. data/lib/insika/commands/resolve_proposal.rb +108 -0
  49. data/lib/insika/commands/revoke_contact.rb +49 -0
  50. data/lib/insika/commands/rollback_harvest.rb +86 -0
  51. data/lib/insika/commands/run_distillation.rb +186 -0
  52. data/lib/insika/commands/run_harvest.rb +393 -0
  53. data/lib/insika/commands/send_message.rb +103 -6
  54. data/lib/insika/commands/session_purge.rb +67 -0
  55. data/lib/insika/contact_store.rb +183 -0
  56. data/lib/insika/context/builder.rb +21 -3
  57. data/lib/insika/context/fragment.rb +7 -3
  58. data/lib/insika/context/priority.rb +3 -0
  59. data/lib/insika/context/provider.rb +17 -3
  60. data/lib/insika/context/providers/briefing.rb +96 -0
  61. data/lib/insika/context/providers/memory.rb +16 -7
  62. data/lib/insika/context/providers/prompt.rb +30 -2
  63. data/lib/insika/context/providers/skill.rb +2 -0
  64. data/lib/insika/context/providers/tool_search.rb +2 -0
  65. data/lib/insika/context_trace_store.rb +38 -2
  66. data/lib/insika/distill.rb +224 -0
  67. data/lib/insika/distill_engine.rb +169 -0
  68. data/lib/insika/doctor.rb +716 -6
  69. data/lib/insika/dsl/runtime.rb +9 -3
  70. data/lib/insika/dsl/server_boot.rb +70 -1
  71. data/lib/insika/dsl.rb +83 -3
  72. data/lib/insika/edge_limiter.rb +28 -7
  73. data/lib/insika/env_schema.rb +9 -2
  74. data/lib/insika/errors.rb +11 -0
  75. data/lib/insika/evals/assertions.rb +3 -2
  76. data/lib/insika/evals/pairwise.rb +18 -6
  77. data/lib/insika/evidence.rb +183 -0
  78. data/lib/insika/executor.rb +687 -70
  79. data/lib/insika/followup_engine.rb +207 -0
  80. data/lib/insika/followup_policy.rb +221 -0
  81. data/lib/insika/followup_store.rb +306 -0
  82. data/lib/insika/funnel_declaration.rb +106 -0
  83. data/lib/insika/funnel_fold.rb +179 -0
  84. data/lib/insika/funnel_store.rb +163 -0
  85. data/lib/insika/golden_store.rb +1 -1
  86. data/lib/insika/grounding/matcher.rb +69 -0
  87. data/lib/insika/grounding.rb +44 -0
  88. data/lib/insika/harvest/conversion_gate.rb +159 -0
  89. data/lib/insika/harvest/criterion.rb +98 -0
  90. data/lib/insika/harvest/gate.rb +194 -0
  91. data/lib/insika/harvest/negative_list.rb +199 -0
  92. data/lib/insika/harvest.rb +241 -0
  93. data/lib/insika/harvest_engine.rb +193 -0
  94. data/lib/insika/harvest_store.rb +548 -0
  95. data/lib/insika/media.rb +298 -0
  96. data/lib/insika/memory_audit_store.rb +85 -0
  97. data/lib/insika/memory_store.rb +264 -23
  98. data/lib/insika/message_origin.rb +6 -1
  99. data/lib/insika/model_visible.rb +87 -0
  100. data/lib/insika/model_visible_trace_store.rb +66 -0
  101. data/lib/insika/onboarding.rb +6 -1
  102. data/lib/insika/outbox_store.rb +42 -4
  103. data/lib/insika/outcome_store.rb +147 -0
  104. data/lib/insika/packaging.rb +163 -0
  105. data/lib/insika/parity/criterion.rb +79 -0
  106. data/lib/insika/parity/verdict.rb +318 -0
  107. data/lib/insika/prefix_fingerprint.rb +58 -0
  108. data/lib/insika/profile_source.rb +22 -1
  109. data/lib/insika/proposal_store.rb +271 -0
  110. data/lib/insika/queue_policy.rb +4 -1
  111. data/lib/insika/refinement/proposer.rb +1 -1
  112. data/lib/insika/reliability.rb +32 -6
  113. data/lib/insika/retention.rb +281 -0
  114. data/lib/insika/routing.rb +101 -0
  115. data/lib/insika/safety/config.rb +44 -4
  116. data/lib/insika/safety/corpus.rb +255 -0
  117. data/lib/insika/safety/detectors.rb +32 -113
  118. data/lib/insika/safety/factory.rb +15 -2
  119. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  120. data/lib/insika/safety/grounding_validator.rb +49 -0
  121. data/lib/insika/safety/input_guardrail.rb +1 -1
  122. data/lib/insika/safety/moderator.rb +1 -1
  123. data/lib/insika/safety/output_filter.rb +9 -5
  124. data/lib/insika/safety/output_validator.rb +11 -5
  125. data/lib/insika/schema_guard.rb +35 -0
  126. data/lib/insika/server/app.rb +144 -4
  127. data/lib/insika/server/rack_app.rb +16 -0
  128. data/lib/insika/server/responses.rb +40 -1
  129. data/lib/insika/session_actor.rb +8 -4
  130. data/lib/insika/session_store.rb +65 -2
  131. data/lib/insika/settings_store.rb +10 -0
  132. data/lib/insika/shadow_pair_store.rb +258 -0
  133. data/lib/insika/skill_catalog.rb +4 -0
  134. data/lib/insika/soak/envelope.rb +140 -0
  135. data/lib/insika/soak/report.rb +392 -0
  136. data/lib/insika/soak/runner.rb +554 -0
  137. data/lib/insika/store.rb +10 -1
  138. data/lib/insika/stores/memory.rb +6 -0
  139. data/lib/insika/stores/sqlite.rb +8 -0
  140. data/lib/insika/studio/app.rb +875 -14
  141. data/lib/insika/studio/assets/dist/application.css +1 -1
  142. data/lib/insika/studio/assets/dist/application.js +22 -22
  143. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  144. data/lib/insika/studio/forms.rb +262 -5
  145. data/lib/insika/studio/nav_icons.rb +6 -1
  146. data/lib/insika/studio/views/agent_detail.erb +627 -84
  147. data/lib/insika/studio/views/agents.erb +10 -6
  148. data/lib/insika/studio/views/approvals.erb +4 -1
  149. data/lib/insika/studio/views/chats.erb +4 -1
  150. data/lib/insika/studio/views/customer.erb +94 -0
  151. data/lib/insika/studio/views/customers.erb +32 -0
  152. data/lib/insika/studio/views/evals.erb +4 -1
  153. data/lib/insika/studio/views/facts.erb +133 -0
  154. data/lib/insika/studio/views/followups.erb +125 -0
  155. data/lib/insika/studio/views/funnel.erb +106 -0
  156. data/lib/insika/studio/views/harvest.erb +234 -0
  157. data/lib/insika/studio/views/home.erb +2 -1
  158. data/lib/insika/studio/views/layout.erb +1 -0
  159. data/lib/insika/studio/views/parity.erb +147 -0
  160. data/lib/insika/studio/views/playground.erb +7 -1
  161. data/lib/insika/studio/views/session.erb +56 -1
  162. data/lib/insika/studio/views/settings.erb +2 -0
  163. data/lib/insika/studio/views/skills.erb +0 -1
  164. data/lib/insika/studio/views/system_files.erb +1 -1
  165. data/lib/insika/studio/views/task.erb +13 -0
  166. data/lib/insika/studio/views/tasks.erb +4 -1
  167. data/lib/insika/studio/views/tools.erb +0 -1
  168. data/lib/insika/task_store.rb +21 -1
  169. data/lib/insika/testing/store_contract.rb +27 -6
  170. data/lib/insika/tick.rb +25 -1
  171. data/lib/insika/tool_definition.rb +17 -7
  172. data/lib/insika/tool_envelope.rb +69 -0
  173. data/lib/insika/tool_manifest.rb +5 -1
  174. data/lib/insika/tools/data_defined_tool.rb +10 -0
  175. data/lib/insika/tools/generate_image.rb +44 -0
  176. data/lib/insika/tools/schedule_followup.rb +164 -0
  177. data/lib/insika/tools/tts.rb +47 -0
  178. data/lib/insika/tools/update_briefing.rb +126 -0
  179. data/lib/insika/turn_state.rb +38 -1
  180. data/lib/insika/turn_timing.rb +22 -2
  181. data/lib/insika/version.rb +1 -1
  182. data/lib/insika/vitals.rb +84 -0
  183. data/lib/insika/wiring/graph.rb +300 -13
  184. data/lib/insika.rb +92 -3
  185. metadata +77 -1
@@ -6,6 +6,10 @@ require_relative "moderator"
6
6
  require_relative "output_filter"
7
7
  require_relative "input_guardrail"
8
8
  require_relative "output_validator"
9
+ # the grounding validator (flagged as the OutputValidator's first
10
+ # step) and the enforcer (the executor's stage-8 boundary call). Pure Ruby.
11
+ require_relative "grounding_validator"
12
+ require_relative "grounding_enforcer"
9
13
 
10
14
  module Insika
11
15
  module Safety
@@ -36,15 +40,24 @@ module Insika
36
40
  end
37
41
 
38
42
  # The after_task hook (register with hooks.register(:task, after: ...)).
43
+ # the grounding validator is the OutputValidator's FIRST step —
44
+ # it runs before the output gate (D9), so grounding is independent of the
45
+ # guardrails opt-in.
39
46
  def output_validator
40
- OutputValidator.new(ask_factory: ->(config) { ask_for(config) })
47
+ OutputValidator.new(ask_factory: ->(config) { ask_for(config) },
48
+ grounding: GroundingValidator.new)
41
49
  end
42
50
 
51
+ # the `:enforce` boundary step the Executor calls between
52
+ # stages 6 and 8. Inert unless the profile's grounding.mode is :enforce.
53
+ def grounding_enforcer = GroundingEnforcer.new
54
+
43
55
  # Injected into the Executor. ->(state) { OutputFilter | nil }: a fresh stateful
44
56
  # filter per turn when the agent has output guardrails on; nil = off (parity).
45
57
  def content_filter_factory
46
58
  lambda do |state|
47
- Config.from_profile(state.profile).output ? OutputFilter.new : nil
59
+ config = Config.from_profile(state.profile)
60
+ config.output ? OutputFilter.new(corpus: config.corpus) : nil
48
61
  end
49
62
  end
50
63
 
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ module Safety
5
+ # — the `:enforce` half of grounding: the claim's SENTENCE is
6
+ # cut from the content the turn persists and delivers. A NEW stage-8 boundary
7
+ # step (a plain executor call, NOT an after_task hook — after_task fires too
8
+ # late to change what is persisted).
9
+ #
10
+ # The cut removes the SENTENCE that contains an ungrounded claim (never a
11
+ # surgical span — removing half a sentence produces broken prose). What the
12
+ # cut CANNOT do is unsay a streamed delta — for a streaming surface `:enforce`
13
+ # still delivers the cut text via channel delivery and flags the turn; that
14
+ # honesty is why the own default is `:flag` (ship `:enforce` only after
15
+ # the matcher audit). Runs ONLY in :enforce mode — inert otherwise.
16
+ class GroundingEnforcer
17
+ # -> [new_content, state]. The stage-8 boundary call. Fail-open: grounding
18
+ # must never break a completed turn.
19
+ def call(task, state, content)
20
+ grounding = grounding_of(state)
21
+ return [content, state] unless grounding&.enforce?
22
+
23
+ ledger = state.respond_to?(:evidence_ledger) ? state.evidence_ledger : nil
24
+ return [content, state] unless ledger
25
+
26
+ text = content.to_s
27
+ refs = grounding.matcher.references(text)
28
+ claims = grounding.matcher.ungrounded(refs, evidence_ids: ledger.ids)
29
+ return [content, state] if claims.empty?
30
+
31
+ cut = cut_sentences(text, claims)
32
+ claims.each { |c| ledger.ungrounded_count(c) }
33
+ state.guardrail_flags = Array(state.guardrail_flags) + [{
34
+ category: "ungrounded", source: "evidence", action: "cut",
35
+ detail: "cut claim(s): #{claims.join(', ')}" }]
36
+ [cut, state]
37
+ rescue StandardError
38
+ [content, state] # fail-open: grounding must never break a completed turn
39
+ end
40
+
41
+ private
42
+
43
+ def grounding_of(state)
44
+ raw = state.profile.respond_to?(:grounding) ? state.profile.grounding : nil
45
+ raw && Insika::Grounding.parse(raw)
46
+ end
47
+
48
+ # The smallest self-contained unit: split on sentence boundaries and drop
49
+ # every sentence containing an ungrounded reference. Conservative on
50
+ # purpose — a sentence with TWO claims, one grounded and one not, is cut
51
+ # whole (documented behavior).
52
+ def cut_sentences(text, claims)
53
+ sentences = text.split(/(?<=[.!?])\s+/)
54
+ kept = sentences.reject { |sentence| claims.any? { |claim| sentence.include?(claim) } }
55
+ kept.join(" ").strip
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ module Safety
5
+ # — the `:flag` half of grounding. A reply claiming a product
6
+ # whose reference is not in the evidence ledger is FLAGGED (audit via the
7
+ # existing `:guardrail_flagged` channel) — the text is already out, so
8
+ # flagging is audit, exactly like every other output flag.
9
+ #
10
+ # Runs as the FIRST step of the OutputValidator's after_task chain, BEFORE
11
+ # the `config.output` gate (D9): grounding is evidence integrity, independent
12
+ # of the guardrails opt-in — an agent with guardrails off and
13
+ # `grounding.mode: :flag` still gets the check. Fail-open on any error
14
+ # (grounding must never break a completed turn).
15
+ class GroundingValidator
16
+ # -> state (the after_task contract). Appends `:ungrounded` flags.
17
+ def call(state)
18
+ grounding = grounding_of(state)
19
+ # :flag only — :enforce is the enforcer's job, :off / absent is nothing.
20
+ return state if grounding.nil? || grounding.off? || grounding.enforce?
21
+
22
+ ledger = state.respond_to?(:evidence_ledger) ? state.evidence_ledger : nil
23
+ return state unless ledger
24
+
25
+ text = state.response_content.to_s
26
+ return state if text.empty?
27
+
28
+ refs = grounding.matcher.references(text)
29
+ claims = grounding.matcher.ungrounded(refs, evidence_ids: ledger.ids)
30
+ claims.each { |c| ledger.ungrounded_count(c) }
31
+ unless claims.empty?
32
+ state.guardrail_flags = Array(state.guardrail_flags) + [{
33
+ category: "ungrounded", source: "evidence",
34
+ detail: "product claim without tool evidence: #{claims.join(', ')}" }]
35
+ end
36
+ state
37
+ rescue StandardError
38
+ state # fail-open
39
+ end
40
+
41
+ private
42
+
43
+ def grounding_of(state)
44
+ raw = state.profile.respond_to?(:grounding) ? state.profile.grounding : nil
45
+ raw && Insika::Grounding.parse(raw)
46
+ end
47
+ end
48
+ end
49
+ end
@@ -33,7 +33,7 @@ module Insika
33
33
  config = Config.from_profile(state.profile)
34
34
  return nxt.call(state) unless config.input
35
35
 
36
- hit = Detectors.scan_input(state.message.to_s, categories: config.input_categories)
36
+ hit = Detectors.scan_input(state.message.to_s, categories: config.input_categories, corpus: config.corpus)
37
37
  return block(state, config, category: hit[:category], source: :deterministic, detail: hit[:matched]) if hit
38
38
 
39
39
  if config.moderator? && (mod = build_moderator(config))
@@ -13,7 +13,7 @@ module Insika
13
13
  #
14
14
  # FAIL-OPEN by construction: the deterministic layer already ran and caught the
15
15
  # gross cases, so an unparseable/failed moderator reply must NOT block a
16
- # legitimate customer. Blocking is the high-stakes direction (RFC: a false
16
+ # legitimate customer. Blocking is the high-stakes direction (a false
17
17
  # positive turns away a real buyer). But fail-open is not a fake negative
18
18
  # a moderator that could not answer returns the third state
19
19
  # `unavailable` — it does not block, and it is NOT recorded as a clean `allow`,
@@ -9,7 +9,7 @@ module Insika
9
9
  # sits between the provider chunks and the `:content` event and only ever emits
10
10
  # text it has PROVEN clean.
11
11
  #
12
- # The hard part (RFC, "fronteira de chunk"): a CPF/secret can arrive split
12
+ # The hard part ("fronteira de chunk"): a CPF/secret can arrive split
13
13
  # across two chunks and neither half matches on its own. A naive delta-by-delta
14
14
  # regex leaks "by construction". So the filter keeps a SLIDING BUFFER: it retains
15
15
  # the tail that might still be growing into a match (Detectors::OPEN_TAIL, which
@@ -21,7 +21,11 @@ module Insika
21
21
  class OutputFilter
22
22
  attr_reader :redaction_counts
23
23
 
24
- def initialize
24
+ # `corpus` : the compiled corpus for the turn — the
25
+ # language-filtered PII set and its open-tail pattern. Default = the
26
+ # full shipped corpus (parity).
27
+ def initialize(corpus: Corpus.compile)
28
+ @corpus = corpus
25
29
  @buf = +""
26
30
  @emitted = +"" # full redacted text emitted so far (for the persisted content)
27
31
  @redaction_counts = Hash.new(0)
@@ -51,11 +55,11 @@ module Insika
51
55
  def safe_cut
52
56
  cut = @buf.length
53
57
 
54
- if (m = @buf.match(Detectors::OPEN_TAIL))
58
+ if (m = @buf.match(@corpus.open_tail))
55
59
  cut = [cut, m.begin(0)].min
56
60
  end
57
61
 
58
- Detectors.match_ranges(@buf).each do |b, e|
62
+ @corpus.match_ranges(@buf).each do |b, e|
59
63
  cut = b if b < cut && cut < e
60
64
  end
61
65
 
@@ -69,7 +73,7 @@ module Insika
69
73
 
70
74
  chunk = @buf[0...cut]
71
75
  @buf = @buf[cut..] || +""
72
- redacted, counts = Detectors.redact(chunk)
76
+ redacted, counts = @corpus.redact(chunk)
73
77
  counts.each { |name, n| @redaction_counts[name] += n }
74
78
  @emitted << redacted
75
79
  redacted
@@ -27,13 +27,19 @@ module Insika
27
27
  class OutputValidator
28
28
  # `ask_factory` (optional): ->(config) { ->(prompt){text} | nil }, built by the
29
29
  # Safety::Factory from the utility_model. nil = deterministic only.
30
- def initialize(ask_factory: nil)
30
+ # `grounding` (optional): a GroundingValidator — its step runs
31
+ # FIRST in #call, before the `config.output` gate (D9: grounding is
32
+ # evidence integrity, independent of the guardrails opt-in).
33
+ def initialize(ask_factory: nil, grounding: nil)
31
34
  @ask_factory = ask_factory
35
+ @grounding = grounding
32
36
  end
33
37
 
34
38
  # after_task hook body. Idempotent and defensive: never raises out (a hook
35
- # error must not fail a committed turn).
39
+ # error must not fail a committed turn). Grounding runs BEFORE the output
40
+ # gate so an agent with guardrails off and grounding on still gets the check.
36
41
  def call(state)
42
+ state = @grounding&.call(state) || state
37
43
  config = Config.from_profile(state.profile)
38
44
  return state unless config.output
39
45
 
@@ -41,7 +47,7 @@ module Insika
41
47
  return state if text.empty?
42
48
 
43
49
  flags = []
44
- flags.concat(deterministic_flags(text))
50
+ flags.concat(deterministic_flags(text, config.corpus))
45
51
  flags.concat(llm_flags(text, config)) if config.moderator?
46
52
 
47
53
  state.guardrail_flags = Array(state.guardrail_flags) + flags unless flags.empty?
@@ -55,8 +61,8 @@ module Insika
55
61
  # Residual PII/secret that somehow reached the final text (the stream filter
56
62
  # should have caught it — this is defense in depth, and the flag itself
57
63
  # carries category counts, never the raw value).
58
- def deterministic_flags(text)
59
- _redacted, counts = Detectors.redact(text)
64
+ def deterministic_flags(text, corpus)
65
+ _redacted, counts = Detectors.redact(text, corpus: corpus)
60
66
  return [] if counts.empty?
61
67
 
62
68
  [{ category: "pii_residual", source: "deterministic", detail: counts.map { |k, v| "#{k}:#{v}" }.join(",") }]
@@ -131,6 +131,41 @@ module Insika
131
131
  end
132
132
  end
133
133
 
134
+ # Walks a dotted path on a plain object: nil when a segment
135
+ # is absent or the intermediate is not a Hash. Shared by the evidence output
136
+ # check and the Processor's extraction.
137
+ def dig(obj, path)
138
+ path.to_s.split(".").reduce(obj) do |cur, seg|
139
+ return nil unless cur.is_a?(Hash) && cur.key?(seg)
140
+
141
+ cur[seg]
142
+ end
143
+ end
144
+
145
+ # The evidence RESULT contract: {items: [{id, line}]} .
146
+ # -> nil | String. Same idiom as `violation`: nil = fine, one message = what
147
+ # is wrong. A malformed evidence result is a correctable TOOL answer — the
148
+ # envelope returns it to the model as `{error:}`, exactly like a malformed
149
+ # call is today. A raw body that is NOT an object (a bare JSON array from a
150
+ # search, a string, nil) is a violation — never a silent `{items: []}` that
151
+ # the model reads as "no products".
152
+ def violation_output(spec, raw)
153
+ return nil if spec.nil?
154
+ return "evidence: result must be an object" unless raw.is_a?(Hash)
155
+
156
+ items = dig(raw, spec.items_path)
157
+ return "evidence: items is missing" if items.nil?
158
+ return "evidence: items must be a list" unless items.is_a?(Array)
159
+
160
+ items.each_with_index do |item, i|
161
+ ok = item.is_a?(Hash) &&
162
+ Coercion.present?(item["id"] || item[:id]) &&
163
+ (item["line"] || item[:line]).is_a?(String)
164
+ return "evidence: items[#{i}] must be {id, line}" unless ok
165
+ end
166
+ nil
167
+ end
168
+
134
169
  # Name the shape the way a model reads it, not the way Ruby does.
135
170
  def kind(value)
136
171
  case value
@@ -55,7 +55,8 @@ module Insika
55
55
  def initialize(command_bus:, event_stream:, session_store:, task_store:,
56
56
  config:, pending_action_store: nil, a2a: nil, provisioner: nil,
57
57
  workflow_registry: nil, onboarding: nil, profiles: nil,
58
- channels: nil, logger: nil, token_store: nil)
58
+ channels: nil, logger: nil, token_store: nil, outcome_store: nil,
59
+ executor: nil, db_path: nil)
59
60
  @command_bus = command_bus
60
61
  @event_stream = event_stream
61
62
  @session_store = session_store
@@ -91,10 +92,18 @@ module Insika
91
92
  # injecting the registry (nil ⇒ the routes do not exist, parity with @a2a).
92
93
  # The channel does the translating; this class keeps doing only transport.
93
94
  @channels = channels
95
+ # WS7: business outcomes per conversation (POST /v1/outcomes — a Control
96
+ # command on the bus — and the tenant-scoped GET /v1/outcomes read).
97
+ # nil = the routes 404 (parity).
98
+ @outcome_store = outcome_store
94
99
  # where a 500's error_ref goes to be FOUND. nil = silent (parity for
95
100
  # embedders); the serving wirings pass $stdout. Class+message+backtrace
96
101
  # only — the ref never travels with request payloads (secrets stay out).
97
102
  @logger = logger
103
+ # GET /v1/vitals: process readings, operator-only, no store.
104
+ # Both optional — nil means the body simply omits those readings.
105
+ @executor = executor
106
+ @db_path = db_path
98
107
  @heartbeat = config.fetch(:heartbeat, 15)
99
108
  @sync_timeout = config.fetch(:sync_timeout, 10) # synchronous control
100
109
  end
@@ -166,6 +175,10 @@ module Insika
166
175
  handle_trigger_workflow(req, name)
167
176
  in ["POST", ["v1", "responses"]]
168
177
  handle_responses(req)
178
+ in ["POST", ["v1", "outcomes"]] if @outcome_store
179
+ handle_record_outcome(req)
180
+ in ["GET", ["v1", "outcomes"]] if @outcome_store
181
+ handle_list_outcomes(req)
169
182
  in ["POST", ["v1", "tools", "manifest"]]
170
183
  handle_import_tools(req)
171
184
  in ["POST", ["v1", "mcp", name, "import"]]
@@ -182,8 +195,12 @@ module Insika
182
195
  handle_read_task(req, id)
183
196
  in ["GET", ["v1", "events"]]
184
197
  handle_events(req)
198
+ in ["GET", ["v1", "vitals"]]
199
+ handle_vitals
185
200
  in ["POST", ["channels", id, "events"]] if @channels
186
201
  handle_channel_event(req, id)
202
+ in ["POST", ["channels", id, "shadow-reply"]] if @channels
203
+ handle_channel_shadow_reply(req, id)
187
204
  in ["POST", ["channels", id, "sessions"]] if @channels
188
205
  handle_channel_session(req, id)
189
206
  in ["POST", ["channels", id, "messages"]] if @channels
@@ -241,7 +258,7 @@ module Insika
241
258
  # tomorrow is gated by default and publishing it is a deliberate edit here.
242
259
  def channel_route?(method, segments)
243
260
  case [method, segments]
244
- in ["POST", ["channels", _, "events" | "sessions" | "messages"]] then true
261
+ in ["POST", ["channels", _, "events" | "sessions" | "messages" | "shadow-reply"]] then true
245
262
  in ["GET", ["channels", _, "asset", _]] then true
246
263
  in ["OPTIONS", ["channels", _, *]] then true # CORS preflight carries no credential, by spec
247
264
  else false
@@ -259,11 +276,44 @@ module Insika
259
276
  # POST /v1/commands/:type — generic: every new Command is born with a
260
277
  # transport. The control vs turn distinction is BY THE SHAPE of the result (the
261
278
  # transport knows no semantics).
279
+ #
280
+ # The principal's tenant is stamped like on every other surface. It is
281
+ # nil for an operator (this route is operator-only — see TENANT_SURFACES),
282
+ # which is exactly why a tenant-scoped command such as `forget_customer`
283
+ # ALSO reads a `tenant` from its payload: over HTTP the operator names the
284
+ # tenant, because the credential does not carry one.
262
285
  def handle_command(req, type)
263
- command = Insika::Command.build(type.to_sym, parse_body(req), transport: :http)
286
+ command = Insika::Command.build(type.to_sym, parse_body(req), transport: :http,
287
+ tenant: req_tenant(req))
264
288
  command_response(dispatch_with_timeout(command))
265
289
  end
266
290
 
291
+ # POST /v1/outcomes — the operator or the integration records a business
292
+ # outcome for a conversation (WS7). A Control command on the bus, tenant
293
+ # stamped from the principal (WS1) — additive, outside the response
294
+ # contract: the engine transports the outcome, never interprets it.
295
+ # 201 { outcome: { agent, outcome, value, session_id, at } }.
296
+ def handle_record_outcome(req)
297
+ command = Insika::Command.build(:record_outcome, parse_body(req), transport: :http,
298
+ tenant: req_tenant(req))
299
+ record = dispatch_with_timeout(command)
300
+ json_response(201, { outcome: { agent: record.agent, outcome: record.outcome,
301
+ value: record.value, session_id: record.session_id,
302
+ at: record.at } })
303
+ end
304
+
305
+ # GET /v1/outcomes[?agent=] — the Studio scorecard's data: the LAST
306
+ # outcome per agent (state cards) + the per-day series. Tenant-scoped
307
+ # (WS1): a tenant principal reads only its own outcomes.
308
+ def handle_list_outcomes(req)
309
+ tenant = req_tenant(req)
310
+ agent = req.GET["agent"]
311
+ latest = @outcome_store.latest_per_agent(tenant: tenant)
312
+ latest = { agent => latest[agent] }.compact if agent && !agent.empty?
313
+ series = @outcome_store.series(tenant: tenant, agent: agent)
314
+ json_response(200, { latest: latest, series: series })
315
+ end
316
+
267
317
  # POST /v1/sessions — sugar for create_session; 201 {session}.
268
318
  def handle_create_session(req)
269
319
  body = parse_body(req)
@@ -350,6 +400,10 @@ module Insika
350
400
  ensure_session(parsed[:user], tenant: tenant)
351
401
  payload = { agent: parsed[:agent], session_id: parsed[:user], message: parsed[:message] }
352
402
  payload[:origin] = parsed[:origin] if parsed[:origin] # declared, else absent
403
+ payload[:customer] = parsed[:customer] if parsed[:customer] # WS8: the memory scope handle
404
+ payload[:parts] = parsed[:parts] if parsed[:parts] # WS9: multimodal content parts
405
+ payload[:source] = parsed[:source] if parsed[:source] # WS9: voice-marked text
406
+ payload[:channel] = parsed[:channel] if parsed[:channel] # WS9 (saída): the channel's output capabilities
353
407
  message_flow(payload, stream: true, serialize: Responses.method(:frame_for),
354
408
  tenant: tenant)
355
409
  end
@@ -481,6 +535,8 @@ TENANT_SURFACES = [
481
535
  ["POST", ["v1", "sessions"]],
482
536
  ["POST", ["v1", "messages"]],
483
537
  ["POST", ["v1", "responses"]],
538
+ ["POST", ["v1", "outcomes"]],
539
+ ["GET", ["v1", "outcomes"]],
484
540
  ["POST", ["v1", "workflows", nil]],
485
541
  ["GET", ["v1", "workflows"]],
486
542
  ["GET", ["v1", "sessions", nil]],
@@ -535,7 +591,52 @@ TENANT_SURFACES = [
535
591
 
536
592
  payload = { agent: parsed[:agent], session_id: session_id,
537
593
  message: parsed[:message], event_id: parsed[:event_id] }.compact
538
- channel_ack(payload, transport: :"channel:#{id}")
594
+ ack = channel_ack(payload, transport: :"channel:#{id}")
595
+ # in shadow the mirror's own reply may ride the SAME call
596
+ # (Shape 1) — recorded before we answer, through the one command.
597
+ if channel.respond_to?(:shadow?) && channel.shadow?
598
+ dispatch_shadow_reply(id, parsed) if parsed[:incumbent_reply]
599
+ return shadow_ack(ack)
600
+ end
601
+ ack
602
+ end
603
+
604
+ # POST /channels/:id/shadow-reply — the mirror contract's fallback (Shape 2),
605
+ # for a consumer that mirrors the exchange BEFORE it answers the customer.
606
+ # Same channel_gate (auth is the channel's, not the route's); 404 unless the
607
+ # channel is in shadow — the same parity every other optional surface has.
608
+ def handle_channel_shadow_reply(req, id)
609
+ channel = @channels.find(id)
610
+ return not_found if channel.nil? || !(channel.respond_to?(:shadow?) && channel.shadow?)
611
+
612
+ gate = channel_gate(channel, req)
613
+ return gate if gate
614
+
615
+ parsed = channel.parse_shadow_reply(req, body: parse_raw_body(req)) # ValidationError -> 422
616
+ result = dispatch_shadow_reply(id, parsed)
617
+ json_response(202, { pair_id: result[:pair_id], status: result[:status] })
618
+ end
619
+
620
+ # The incumbent half rides ONE command whatever shape it arrived in, so
621
+ # server/ never writes to a store and both doors share one behaviour.
622
+ def dispatch_shadow_reply(id, parsed)
623
+ @command_bus.dispatch(
624
+ Insika::Command.build(:record_shadow_reply,
625
+ { channel: id.to_s, external_id: parsed[:external_id],
626
+ event_id: parsed[:event_id], reply: parsed[:incumbent_reply] || parsed[:reply],
627
+ at: parsed[:at] }.compact,
628
+ transport: :"channel:#{id}")
629
+ )
630
+ end
631
+
632
+ # 202 -> 200 {task_id, shadow: true}: a consumer wired to "202 means a
633
+ # reply is coming" must not be silently misled. The duplicate/merged/
634
+ # steered verdicts pass through untouched (they already say 200).
635
+ def shadow_ack(ack)
636
+ status, _headers, body = ack
637
+ return ack unless status == 202
638
+
639
+ json_response(200, JSON.parse(body.join).merge("shadow" => true))
539
640
  end
540
641
 
541
642
  # POST /channels/:id/sessions — mint a conversation for a PUBLIC Shape A
@@ -780,6 +881,36 @@ TENANT_SURFACES = [
780
881
  # happens here, BEFORE the SSE opens -> closes the subscription and propagates to the
781
882
  # #call rescue (becomes an HTTP status).
782
883
  def message_flow(payload, stream:, serialize: nil, transport: :http, tenant: nil)
884
+ # WS9: content parts are CONTRACT at the edge — the closed shape is
885
+ # validated here (422) before the command is built; the engine stays
886
+ # lenient for transports that bypass this surface.
887
+ parts = payload[:parts] || payload["parts"]
888
+ if parts && !Insika::Media.well_formed?(parts)
889
+ raise Insika::ValidationError,
890
+ "malformed content part — each part must be {type: text|image|audio} with text/url"
891
+ end
892
+ # WS9: the only declared message source is "voice" (pre-transcribed
893
+ # audio) — a consumer that writes prose here is told so, not silently.
894
+ source = payload[:source] || payload["source"]
895
+ if source && source.to_s != "voice"
896
+ raise Insika::ValidationError, 'source must be "voice"'
897
+ end
898
+ # WS9 (saída): the channel's declared OUTPUT media capabilities. The
899
+ # closed list IS the "abstraction admits only what leaks" rule: an
900
+ # unknown value is refused here (422), never silently ignored; a
901
+ # declared value is what the executor may wire (generate_image/tts).
902
+ channel = payload[:channel] || payload["channel"]
903
+ if channel.is_a?(Hash)
904
+ caps = Insika::Media.channel_capabilities(channel)
905
+ unknown = caps - Insika::Media::OUTPUT_CAPABILITIES
906
+ unless unknown.empty?
907
+ raise Insika::ValidationError,
908
+ "unknown channel capability: #{unknown.join(', ')}"
909
+ end
910
+
911
+ payload = payload.dup
912
+ payload[:channel] = { capabilities: caps }
913
+ end
783
914
  # WS1: a tenant's session_id is NAMESPACED before the command is built,
784
915
  # so the turn lands on the tenant's OWN session even when another tenant
785
916
  # uses the same chat id. The payload the caller sent is untouched.
@@ -993,6 +1124,15 @@ TENANT_SURFACES = [
993
1124
  # touch a store (health cannot fail on IO nor require auth).
994
1125
  def health = json_response(200, { status: "ok" })
995
1126
 
1127
+ # process vitals. Operator-only by construction: not in
1128
+ # PUBLIC_ROUTES (no bearer -> unauthorized) and not in TENANT_SURFACES
1129
+ # (a tenant principal hits the operator-surface gate), so only an
1130
+ # operator reads process internals. Reads no store — pure OS/VM
1131
+ # readings, safe at any rate.
1132
+ def handle_vitals
1133
+ json_response(200, Insika::Vitals.snapshot(executor: @executor, db_path: @db_path))
1134
+ end
1135
+
996
1136
  # Thin Subscription decorator: discards
997
1137
  # events from OTHER tasks and CLOSES the subscription after forwarding the task's
998
1138
  # terminal event. Solves the subscribe-before-task_id gap without touching
@@ -42,6 +42,10 @@ module Insika
42
42
  end
43
43
 
44
44
  attr_reader :token
45
+ attr_reader :graph
46
+ # the frozen criterion, memoized when the relay runs in shadow —
47
+ # nil otherwise. Loaded ONCE at boot (the refusal below), then shared.
48
+ attr_reader :criterion
45
49
 
46
50
  def app
47
51
  tenancy = @config[:tenancy] || ENV["INSIKA_TENANCY"] || "single_tenant"
@@ -71,6 +75,7 @@ module Insika
71
75
  channels: (@graph.channel_registry if channels?),
72
76
  config: { gateway_token: @token, tenancy: tenancy }.merge(@config),
73
77
  token_store: store,
78
+ outcome_store: @graph.outcome_store,
74
79
  # a 500's error_ref must be findable in the process log.
75
80
  logger: $stdout
76
81
  )
@@ -87,6 +92,17 @@ module Insika
87
92
  allow_http: Insika::EnvSchema.truthy?(ENV["INSIKA_EGRESS_ALLOW_HTTP"]),
88
93
  allow_private: Insika::EnvSchema.truthy?(ENV["INSIKA_EGRESS_ALLOW_PRIVATE"])
89
94
  )
95
+ # NO criterion, no shadow — enforced at boot, not per
96
+ # turn. Raises ConfigError when the file is missing, unparseable, or
97
+ # incomplete; the criterion is memoized here and injected into
98
+ # ChannelDelivery (its sha stamps every pair our half records).
99
+ if relay&.shadow?
100
+ criterion_path = Insika::EnvSchema.read("INSIKA_PARITY_CRITERION")
101
+ raise Insika::ConfigError, "shadow mode requires INSIKA_PARITY_CRITERION (the frozen criterion)" if criterion_path.nil?
102
+
103
+ @criterion = Insika::Parity::Criterion.load(criterion_path)
104
+ @graph.channel_delivery&.criterion_sha = @criterion.sha
105
+ end
90
106
  @graph.channel_registry.register(relay.id, relay) if relay
91
107
 
92
108
  widget = Insika::Channels::Web.from_env(
@@ -40,10 +40,45 @@ module Insika
40
40
  raise Insika::ValidationError, "user missing" if user.strip.empty?
41
41
 
42
42
  message = extract_input(body[:input])
43
- raise Insika::ValidationError, "input empty" if message.strip.empty?
44
43
 
45
44
  out = { agent: agent.strip, user: user, message: message }
46
45
  (origin = Insika::MessageOrigin.parse!(body[:origin])) && (out[:origin] = origin)
46
+ # WS8: the optional customer_key — per-customer memory scope + purge handle
47
+ (customer = Insika::Coercion.presence(body[:customer])) && (out[:customer] = customer)
48
+ # WS9: the multimodal OPENAI shape — `input` as an array of content parts
49
+ # ({type: text/image/audio}) is preserved additively alongside the
50
+ # joined text; a string input stays byte-identical to before. The
51
+ # CONTRACT is enforced here (422) — the engine stays lenient.
52
+ raw = body[:input]
53
+ if raw.is_a?(Array)
54
+ unless Insika::Media.well_formed?(raw)
55
+ raise Insika::ValidationError,
56
+ "malformed content part — each part must be {type: text|image|audio} with text/url"
57
+ end
58
+
59
+ normalized = Insika::Media.parts(raw).map do |p|
60
+ { "type" => p.type, "text" => p.text, "url" => p.url }.compact
61
+ end
62
+ out[:parts] = normalized unless normalized.empty?
63
+ end
64
+ # The turn needs SOMETHING to be about — text, or media the engine will
65
+ # turn into text (a voice note) or show the model (a photo). Checked
66
+ # after the parts are known, because the anchor use case (a WhatsApp
67
+ # voice note, no caption) carries no text at all and joining only the
68
+ # text parts made it a 422 at the door.
69
+ if message.strip.empty? && Array(out[:parts]).none? { |p| p["type"] != "text" }
70
+ raise Insika::ValidationError, "input empty"
71
+ end
72
+ # WS9: `source` marks pre-transcribed voice text; anything else is refused.
73
+ unless body[:source].nil? || body[:source].to_s == "voice"
74
+ raise Insika::ValidationError, 'source must be "voice"'
75
+ end
76
+ out[:source] = body[:source].to_s if Insika::Coercion.presence(body[:source])
77
+ # WS9 (saída): the channel declares which generated media it can
78
+ # RECEIVE ({ capabilities: ["image_output", "audio_output"] }). Additive
79
+ # + additive sibling on the completed frame; the exact capabilities are
80
+ # validated at the boundary (message_flow), not here.
81
+ (channel = body[:channel]) && (out[:channel] = channel)
47
82
  out
48
83
  end
49
84
 
@@ -146,6 +181,10 @@ module Insika
146
181
  # the OpenAI-shaped response.use it to run their escalation ("stuck" means
147
182
  # what they decide it means, never the engine's business).
148
183
  (outcome = event.data[:outcome]) && (response[:outcome] = outcome.to_s)
184
+ # WS9 (saída): generated media parts (image/audio clips) ride the
185
+ # completed frame additively next to the text — absent when none were
186
+ # generated. The base64 bytes are the consumer's to render/upload.
187
+ (parts = event.data[:output_parts]) && (response[:output_parts] = parts)
149
188
  sse("response.completed", { type: "response.completed", response: response })
150
189
  end
151
190
 
@@ -40,8 +40,12 @@ module Insika
40
40
  #
41
41
  # `policy` (a QueuePolicy) opens the debounce window for this turn; nil or a
42
42
  # policy without a window behaves exactly as before — dequeued and run at once.
43
- def enqueue(task, profile:, resume_from: nil, policy: nil)
44
- @queue.enqueue([task, profile, resume_from, policy])
43
+ #
44
+ # `timing` is the channel clock a channel turn allocated at 202
45
+ # acceptance and already stamped `:inbound`; it rides the queue so the debounce
46
+ # window and the FIFO wait land INSIDE first_balloon_ms.
47
+ def enqueue(task, profile:, resume_from: nil, policy: nil, timing: nil)
48
+ @queue.enqueue([task, profile, resume_from, policy, timing])
45
49
  task.id
46
50
  end
47
51
 
@@ -99,12 +103,12 @@ module Insika
99
103
 
100
104
  def run_loop
101
105
  loop do
102
- task, profile, resume_from, policy = @queue.dequeue # blocks when empty
106
+ task, profile, resume_from, policy, timing = @queue.dequeue # blocks when empty
103
107
  task = hold_at_the_door(task, policy)
104
108
  @running = true
105
109
  @current_task = task
106
110
  begin
107
- @executor.run_serial(task, profile: profile, resume_from: resume_from)
111
+ @executor.run_serial(task, profile: profile, resume_from: resume_from, timing: timing)
108
112
  rescue StandardError
109
113
  # run_serial already maps turn errors; this rescue is defense: an
110
114
  # unexpected error must NEVER bring down the session loop (Async::Stop <