insika 0.3.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +296 -0
  3. data/README.md +48 -12
  4. data/bin/insika +725 -0
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +116 -406
  7. data/docs/API.md +5 -5
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +137 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +14 -14
  12. data/docs/CONTEXT.md +63 -19
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +87 -10
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +128 -3
  17. data/docs/FACTS.md +3 -3
  18. data/docs/HARVEST.md +5 -6
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +17 -29
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +46 -12
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +222 -0
  26. data/docs/REFINEMENT.md +14 -9
  27. data/docs/RELEASING.md +4 -4
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +5 -5
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +23 -7
  32. data/docs/SKILLS.md +11 -2
  33. data/docs/SOAK.md +3 -3
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +176 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +4 -4
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +8 -5
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +99 -17
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/channel_delivery.rb +1 -1
  62. data/lib/insika/chat_builder.rb +50 -19
  63. data/lib/insika/commands/agent_payload.rb +2 -2
  64. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  65. data/lib/insika/commands/delete_artifact.rb +35 -0
  66. data/lib/insika/commands/delete_concept.rb +34 -0
  67. data/lib/insika/commands/delete_mcp.rb +6 -2
  68. data/lib/insika/commands/delete_tenant_data.rb +15 -3
  69. data/lib/insika/commands/gate_refinement.rb +1 -1
  70. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  71. data/lib/insika/commands/restore_concept.rb +34 -0
  72. data/lib/insika/commands/seed_demo_data.rb +31 -0
  73. data/lib/insika/commands/upsert_mcp.rb +6 -3
  74. data/lib/insika/commands/write_concept.rb +57 -0
  75. data/lib/insika/compaction.rb +196 -0
  76. data/lib/insika/context/builder.rb +6 -2
  77. data/lib/insika/context/fragment.rb +4 -1
  78. data/lib/insika/context/priority.rb +8 -0
  79. data/lib/insika/context/providers/briefing.rb +53 -24
  80. data/lib/insika/context/providers/knowledge.rb +108 -0
  81. data/lib/insika/context/providers/prompt.rb +30 -24
  82. data/lib/insika/context/providers/session.rb +46 -10
  83. data/lib/insika/context_trace_store.rb +11 -1
  84. data/lib/insika/cron.rb +189 -0
  85. data/lib/insika/demo/agent_attrs.rb +43 -0
  86. data/lib/insika/demo/golden_cases.rb +81 -0
  87. data/lib/insika/demo/seeder.rb +336 -0
  88. data/lib/insika/doctor.rb +280 -17
  89. data/lib/insika/dsl/definition.rb +3 -2
  90. data/lib/insika/dsl/runtime.rb +64 -79
  91. data/lib/insika/dsl/server_boot.rb +23 -1
  92. data/lib/insika/dsl/system.rb +10 -2
  93. data/lib/insika/dsl.rb +103 -2
  94. data/lib/insika/env_schema.rb +21 -7
  95. data/lib/insika/evals/golden.rb +41 -4
  96. data/lib/insika/evals/judge.rb +47 -2
  97. data/lib/insika/evals/pairwise.rb +11 -0
  98. data/lib/insika/evals/persona.rb +98 -0
  99. data/lib/insika/evals/runner.rb +9 -0
  100. data/lib/insika/evals/simulator.rb +225 -0
  101. data/lib/insika/evals/transport.rb +84 -2
  102. data/lib/insika/event_stream.rb +10 -0
  103. data/lib/insika/executor.rb +295 -55
  104. data/lib/insika/followup_policy.rb +2 -25
  105. data/lib/insika/golden_store.rb +16 -1
  106. data/lib/insika/grounding/matcher.rb +1 -1
  107. data/lib/insika/knowledge.rb +680 -0
  108. data/lib/insika/knowledge_store.rb +140 -0
  109. data/lib/insika/loop_detector.rb +5 -34
  110. data/lib/insika/mcp_client.rb +94 -0
  111. data/lib/insika/mcp_json.rb +74 -0
  112. data/lib/insika/mcp_live_tool.rb +43 -0
  113. data/lib/insika/mcp_store.rb +98 -26
  114. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  115. data/lib/insika/mcp_tool_registry.rb +100 -0
  116. data/lib/insika/media.rb +115 -31
  117. data/lib/insika/message_origin.rb +1 -1
  118. data/lib/insika/middleware.rb +9 -0
  119. data/lib/insika/onboarding.rb +17 -1
  120. data/lib/insika/outcome_store.rb +1 -1
  121. data/lib/insika/overlay_tool_registry.rb +37 -17
  122. data/lib/insika/packaging.rb +2 -2
  123. data/lib/insika/profile_source.rb +15 -1
  124. data/lib/insika/prompt_catalog.rb +10 -0
  125. data/lib/insika/retention.rb +36 -1
  126. data/lib/insika/router/app.rb +157 -0
  127. data/lib/insika/router/backend_pool.rb +98 -0
  128. data/lib/insika/router/hash_ring.rb +55 -0
  129. data/lib/insika/router/proxy_body.rb +34 -0
  130. data/lib/insika/router/session_key.rb +54 -0
  131. data/lib/insika/router.rb +18 -0
  132. data/lib/insika/schedule.rb +177 -0
  133. data/lib/insika/schedule_engine.rb +314 -0
  134. data/lib/insika/schedule_store.rb +208 -0
  135. data/lib/insika/server/app.rb +105 -15
  136. data/lib/insika/server/rack_app.rb +5 -1
  137. data/lib/insika/server/responses.rb +5 -5
  138. data/lib/insika/session_store.rb +34 -4
  139. data/lib/insika/settings_store.rb +8 -1
  140. data/lib/insika/skill_catalog.rb +12 -0
  141. data/lib/insika/soak/runner.rb +4 -4
  142. data/lib/insika/steer_injector.rb +21 -10
  143. data/lib/insika/studio/app.rb +591 -47
  144. data/lib/insika/studio/assets/dist/application.css +1 -1
  145. data/lib/insika/studio/assets/dist/application.js +21 -21
  146. data/lib/insika/studio/forms.rb +57 -5
  147. data/lib/insika/studio/nav_icons.rb +14 -1
  148. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  149. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  150. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  151. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  152. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  153. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  154. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  155. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  156. data/lib/insika/studio/views/_agents_master.erb +44 -0
  157. data/lib/insika/studio/views/_message.erb +49 -32
  158. data/lib/insika/studio/views/agent_detail.erb +61 -820
  159. data/lib/insika/studio/views/agents.erb +70 -57
  160. data/lib/insika/studio/views/artifact.erb +23 -0
  161. data/lib/insika/studio/views/artifacts.erb +59 -0
  162. data/lib/insika/studio/views/evals.erb +2 -2
  163. data/lib/insika/studio/views/facts.erb +1 -1
  164. data/lib/insika/studio/views/funnel.erb +1 -1
  165. data/lib/insika/studio/views/home.erb +106 -67
  166. data/lib/insika/studio/views/knowledge.erb +123 -0
  167. data/lib/insika/studio/views/layout.erb +14 -11
  168. data/lib/insika/studio/views/mcp.erb +174 -80
  169. data/lib/insika/studio/views/session.erb +231 -177
  170. data/lib/insika/studio/views/settings.erb +50 -1
  171. data/lib/insika/studio/views/skills.erb +1 -1
  172. data/lib/insika/studio/views/tools.erb +24 -9
  173. data/lib/insika/telemetry/recorder.rb +49 -1
  174. data/lib/insika/templates/browser-agent/README.md +36 -0
  175. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  176. data/lib/insika/templates/daily-digest/README.md +47 -0
  177. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  178. data/lib/insika/templates/repo-explorer/README.md +36 -0
  179. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  180. data/lib/insika/templates/research-analyst/README.md +26 -0
  181. data/lib/insika/templates/research-analyst/agent.rb +68 -0
  182. data/lib/insika/templates/review-panel/README.md +20 -0
  183. data/lib/insika/templates/review-panel/agent.rb +50 -0
  184. data/lib/insika/templates/travel-planner/README.md +35 -0
  185. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  186. data/lib/insika/templates.rb +112 -0
  187. data/lib/insika/tick.rb +24 -12
  188. data/lib/insika/timezone.rb +45 -0
  189. data/lib/insika/tool_batch.rb +67 -0
  190. data/lib/insika/tool_usage_report.rb +162 -0
  191. data/lib/insika/tools/generate_image.rb +52 -7
  192. data/lib/insika/tools/load_knowledge.rb +74 -0
  193. data/lib/insika/tools/run_persona_eval.rb +328 -0
  194. data/lib/insika/tools/save_artifact.rb +95 -0
  195. data/lib/insika/turn_budget.rb +91 -0
  196. data/lib/insika/turn_output.rb +1 -1
  197. data/lib/insika/turn_state.rb +15 -4
  198. data/lib/insika/version.rb +1 -1
  199. data/lib/insika/wiring/graph.rb +184 -12
  200. data/lib/insika/wiring/graph_chat.rb +102 -0
  201. data/lib/insika.rb +64 -0
  202. metadata +109 -5
  203. data/docs/build.md +0 -14
  204. data/docs/understand.md +0 -10
@@ -0,0 +1,196 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ # In-session compaction: when a session's UNCOMPACTED transcript
5
+ # grows past `compact_after` messages, everything but the last `keep_last`
6
+ # is summarized by a cheap model into one fragment; the tail stays verbatim.
7
+ # This module is the pure half — boundary math, the prompt, and the
8
+ # Summarizer over an injected ask (the Distiller shape: unit-testable
9
+ # without a provider). The trigger lives in the Executor (post-turn, off
10
+ # the critical path); the persistence in SessionStore#set_compaction; the
11
+ # read path in Context::Providers::Session.
12
+ module Compaction
13
+ DEFAULT_KEEP_LAST = 20
14
+ DEFAULT_COMPACT_AFTER = 40
15
+ # A summary that outgrows this is truncated — the compaction must never
16
+ # grow the context it exists to shrink.
17
+ MAX_SUMMARY_CHARS = 6_000
18
+ # Per-message cap in the transcript slice sent to the summarizer (a
19
+ # `role: tool` body can be 4 000 chars in the store); the head of a long
20
+ # result carries the identity of what happened, which is what a summary needs.
21
+ MESSAGE_CHAR_CAP = 1_000
22
+
23
+ # The engine's generic prompt. A platform `compaction.prompt` REPLACES it
24
+ # wholesale (the distill convention) — the engine never writes store
25
+ # vocabulary. The preserve-list is the P28 contract: facts (CEP, order
26
+ # numbers), commitments, the MISSING list, decisions.
27
+ DEFAULT_PROMPT = <<~PROMPT.freeze
28
+ You are compacting the OLD part of an ongoing customer conversation into
29
+ one summary that the assistant will read INSTEAD of those messages. The
30
+ recent messages stay verbatim; your summary is the only surviving trace
31
+ of the old ones — anything you drop is gone for good.
32
+
33
+ Preserve, verbatim where short:
34
+ - every fact the customer stated (sizes, budget, address, postal code/CEP,
35
+ order numbers, product choices, dates, quantities);
36
+ - every commitment the assistant made (promises, prices quoted, delivery
37
+ windows, agreed next steps);
38
+ - what was asked and is still unanswered (the missing information);
39
+ - decisions already made, so nothing gets re-asked or re-litigated.
40
+
41
+ Do not invent, do not editorialize, do not add advice. Answer with the
42
+ summary text only — plain text, compact, in the conversation's own language.
43
+ PROMPT
44
+
45
+ # The compaction plan: summarize messages[from...upto] (from = the previous
46
+ # boundary), keep messages[upto..] verbatim. count = upto - from.
47
+ Plan = Data.define(:from, :upto, :count)
48
+
49
+ module_function
50
+
51
+ # Decides whether (and what) to compact. -> Plan | nil.
52
+ # messages: the session transcript (append-only, RFC-0016).
53
+ # state: the persisted "compaction" hash ({"upto"=>, ...}) | nil.
54
+ # config: the Settings "compaction" hash (keep_last/compact_after).
55
+ # `compact_after` is clamped to at least `keep_last` so the plan always
56
+ # moves the boundary forward. The boundary retreats over `role: "tool"`
57
+ # messages so an eviction unit (assistant-with-tool_calls + its results)
58
+ # is never split — the whole cycle stays verbatim instead.
59
+ def plan(messages:, state:, config:)
60
+ msgs = Array(messages)
61
+ keep_last = positive(config && config["keep_last"], DEFAULT_KEEP_LAST)
62
+ compact_after = positive(config && config["compact_after"], DEFAULT_COMPACT_AFTER)
63
+ compact_after = keep_last if compact_after < keep_last
64
+ from = state ? state["upto"].to_i : 0
65
+ return nil unless msgs.size - from > compact_after
66
+
67
+ upto = msgs.size - keep_last
68
+ upto -= 1 while upto > from && role_of(msgs[upto]) == "tool"
69
+ return nil unless upto > from
70
+
71
+ Plan.new(from: from, upto: upto, count: upto - from)
72
+ end
73
+
74
+ # The full prompt for one compaction run: the base rules, the PREVIOUS
75
+ # summary (so a fact from turn 3 survives every re-compaction — each
76
+ # summary folds the last one in) and only the NEW slice. The slice is
77
+ # sent UNREDACTED on purpose: it replaces transcript the main model
78
+ # already reads raw, inside the same trust boundary, and redaction would
79
+ # delete exactly the facts (CEP, order id) the summary must preserve.
80
+ def prompt(messages:, plan:, previous: nil, base: nil)
81
+ rules = Coercion.presence(base.to_s) || DEFAULT_PROMPT
82
+ parts = [rules.rstrip]
83
+ if Coercion.presence(previous.to_s)
84
+ parts << "## The summary so far (fold it into the new one — its facts must survive)\n\n#{previous}"
85
+ end
86
+ parts << "## The messages to compact\n\n#{transcript(messages, plan)}"
87
+ parts.join("\n\n")
88
+ end
89
+
90
+ # "[i] role: content" over the plan's slice, one line per message; a
91
+ # tool-calling assistant message with no text renders the tool names.
92
+ def transcript(messages, plan)
93
+ Array(messages)[plan.from...plan.upto].to_a.each_with_index.map do |msg, offset|
94
+ "[#{plan.from + offset}] #{role_of(msg)}: #{text_of(msg)}"
95
+ end.join("\n")
96
+ end
97
+
98
+ def role_of(msg) = (msg["role"] || msg[:role]).to_s
99
+
100
+ def text_of(msg)
101
+ content = (msg["content"] || msg[:content]).to_s.strip.gsub(/\s+/, " ")
102
+ if content.empty?
103
+ calls = msg["tool_calls"] || msg[:tool_calls]
104
+ names = Array(calls).filter_map { |c| c.is_a?(Hash) ? (c["name"] || c[:name] || c.dig("function", "name")) : nil }
105
+ content = names.empty? ? "(empty)" : "(tool calls: #{names.join(', ')})"
106
+ end
107
+ content[0, MESSAGE_CHAR_CAP]
108
+ end
109
+
110
+ def positive(value, default)
111
+ n = value.to_i
112
+ n.positive? ? n : default
113
+ end
114
+
115
+ # The one place compaction asks a model for anything. Pure over an
116
+ # injected `ask` (the Distiller shape); the real ask is a lambda built by
117
+ # SummarizerFactory (ruby_llm required lazily, load_guard stays green).
118
+ class Summarizer
119
+ # A blank answer must not overwrite the boundary — empty output is a
120
+ # loud failure, never "the old turns said nothing".
121
+ class Unusable < Insika::ValidationError; end
122
+
123
+ # ask: ->(prompt) { "<raw model text>" } | something answering #content
124
+ # (+ #input_tokens/#output_tokens/#cached_tokens for cost).
125
+ # model: the ref recorded on the event ("utility_model" default).
126
+ attr_reader :model
127
+
128
+ def initialize(ask:, model: "utility_model")
129
+ @ask = ask
130
+ @model = model.to_s
131
+ end
132
+
133
+ # -> { summary: String, cost: { "spent" => N, "cached" => N } | nil }
134
+ # Raises Unusable on a blank answer; truncates past MAX_SUMMARY_CHARS.
135
+ def summarize(prompt:)
136
+ answer = @ask.call(prompt)
137
+ text = Coercion.utf8(text_of(answer)).strip
138
+ raise Unusable, "the summarizer answered with nothing" if text.empty?
139
+
140
+ { summary: text[0, MAX_SUMMARY_CHARS], cost: cost_of(answer) }
141
+ end
142
+
143
+ private
144
+
145
+ def text_of(answer) = (answer.respond_to?(:content) ? answer.content : answer).to_s
146
+
147
+ # nil when the provider said nothing — never 0 (the Distiller's
148
+ # discipline). The cached prefix is INCLUDED in the spent total.
149
+ def cost_of(answer)
150
+ return nil unless answer.respond_to?(:input_tokens) && answer.respond_to?(:output_tokens)
151
+
152
+ input = answer.input_tokens.to_i
153
+ output = answer.output_tokens.to_i
154
+ cached = answer.respond_to?(:cached_tokens) ? answer.cached_tokens.to_i : 0
155
+ { "spent" => input + output, "cached" => cached }
156
+ end
157
+ end
158
+
159
+ # Resolves WHICH model summarizes and builds the ask. compaction.model ->
160
+ # platform utility_model -> nil (nil means "feature inert", never a guess —
161
+ # the DistillerFactory ladder). NOT the fallbacks chain: that one answers
162
+ # "which model serves the customer turn". `ask_factory`/`llm` injectable (specs).
163
+ module SummarizerFactory
164
+ module_function
165
+
166
+ # config: the Settings "compaction" hash. -> Summarizer | nil
167
+ def build(config, utility_model: nil, ask_factory: nil, llm: nil)
168
+ ref = Coercion.presence(config && config["model"]) || Coercion.presence(utility_model)
169
+ return nil if ref.nil?
170
+
171
+ provider, model = split_ref(ref)
172
+ factory = ask_factory || ->(m, p) { ruby_llm_ask(m, p, llm: llm) }
173
+ Summarizer.new(ask: factory.call(model, provider), model: ref)
174
+ end
175
+
176
+ # "provider/model" -> [provider, model]; "model" -> [nil, model] — one
177
+ # syntax for "which model" across features (DistillerFactory's reading).
178
+ def split_ref(ref)
179
+ prov, name = ref.to_s.split("/", 2)
180
+ name ? [prov, name] : [nil, prov]
181
+ end
182
+
183
+ # Temperature 0: the same slice must compact to the same boundary
184
+ # deterministically. `ruby_llm` is required lazily so nothing loads a
185
+ # provider gem until compaction is actually configured (load_guard stays green).
186
+ def ruby_llm_ask(model, provider, llm: nil)
187
+ require "ruby_llm"
188
+ llm ||= RubyLLM
189
+ lambda do |prompt|
190
+ llm.chat(model: model, provider: provider, assume_model_exists: true)
191
+ .with_temperature(0).ask(prompt)
192
+ end
193
+ end
194
+ end
195
+ end
196
+ end
@@ -151,13 +151,17 @@ module Insika
151
151
  identity, volatile = system_frags.partition { |f| (f.layer || :volatile) == :identity }
152
152
  system_frags = sort_canonical(identity) + sort_canonical(volatile)
153
153
  history_frags = fragments.select { |f| f.placement == :history } # production order (chronological)
154
+ # :tail renders after ALL history — the last thing before the current user
155
+ # message. Attention is strongest at the end of the context, so the goal
156
+ # restated here survives a 30-call turn that the head prompt no longer does.
157
+ tail_frags = fragments.select { |f| f.placement == :tail }
154
158
  tool_frags = fragments.select { |f| f.placement == :tool_context }
155
159
 
156
160
  system = system_frags.map(&:content).join("\n\n")
157
- history = history_frags.map(&:content)
161
+ history = history_frags.map(&:content) + tail_frags.map(&:content)
158
162
  tool_context = tool_frags.empty? ? nil : tool_frags.map(&:content).join("\n\n")
159
163
 
160
- canonical = system_frags + history_frags + tool_frags
164
+ canonical = system_frags + history_frags + tail_frags + tool_frags
161
165
  ContextPackage.new(
162
166
  system: system, history: history, tool_context: tool_context,
163
167
  fragments: canonical, budget: { cap: cap, used: canonical.sum(&:tokens), evicted: evicted }
@@ -3,7 +3,10 @@
3
3
  module Insika
4
4
  # Unit of context produced by a provider.
5
5
  # SHARED type (Insika::, not Insika::Context::).
6
- # placement: :system | :history | :tool_context
6
+ # placement: :system | :history | :tail | :tool_context
7
+ # :tail renders AFTER the whole history, i.e. as the last thing
8
+ # the model reads before the current user message. Its content is
9
+ # a message Hash ({role:, content:}), like a :history fragment.
7
10
  # priority: Integer; higher = more important (survives cuts)
8
11
  # tokens: Integer | nil; estimated by the Builder when nil
9
12
  # source: String — provider id (audit)
@@ -17,9 +17,14 @@ module Insika
17
17
  # the injected block.
18
18
  module Priority
19
19
  IDENTITY = 100 # IDENTITY/SOUL (Prompt) — pinned
20
+ RECITATION = 95 # <recitation> the goal restated at the TAIL of context
21
+ # (Briefing). Never cut: it is two lines, and the one
22
+ # turn it gets evicted is the long turn that needed it.
20
23
  PROMPT_REF = 90 # Prompt Catalog guardrails/refs (Prompt) — pinned
21
24
  SKILL_BODY = 85 # <active_skill> trigger-matched body (SkillTrigger)
22
25
  SKILL = 80 # <available_skills> level 1 (Skill)
26
+ KNOWLEDGE = 77 # <knowledge> top-K learned concepts (Knowledge) —
27
+ # below curated skills, above a single conversation's memory
23
28
  MEMORY = 75 # <memory> read path (Memory)
24
29
  TOOL_SEARCH = 70 # <available_tools> level 1 (ToolSearch)
25
30
  BRIEFING = 65 # <briefing> session working state (Briefing) — D5:
@@ -27,6 +32,9 @@ module Insika
27
32
  # pinned prefix), above the turn's own <request_context>.
28
33
  HISTORY_MAX = 79 # history ceiling by recency (Session)
29
34
  HISTORY_BASE = 60 # history base; +idx up to the ceiling (Session)
35
+ COMPACTION = 59 # <conversation_summary> the compacted prefix (RFC-0044)
36
+ # — one step below the oldest verbatim message: under
37
+ # budget it is the "oldest unit" and drops first.
30
38
  REQUEST = 40 # <request_context> — turn injection, the most cuttable
31
39
  end
32
40
  end
@@ -5,9 +5,20 @@ module Insika
5
5
  module Providers
6
6
  # Read path for the session briefing: the per-session
7
7
  # working-state the agent keeps and asks for. Thin adapter over the
8
- # SessionStore, same pattern as Memory: one `:system` fragment,
9
- # deterministic. The MISSING list is rendered, never implied — that list
10
- # is what stops the model re-asking for a field already given.
8
+ # SessionStore, same pattern as Memory, deterministic. The MISSING list is
9
+ # rendered, never implied — that list is what stops the model re-asking
10
+ # for a field already given.
11
+ #
12
+ # TWO fragments, and the split is the whole point:
13
+ # · `<briefing>` (:system) — the DURABLE facts, what is already known.
14
+ # Head of the prompt, where reference material belongs.
15
+ # · `<recitation>` (:tail) — what is still missing and what the next step
16
+ # is, rendered AFTER the whole history, as the last thing before the
17
+ # current user message.
18
+ # Attention is strongest at the END of the context: a goal stated only in
19
+ # the head is the first thing a 30-call turn forgets. The recitation lives
20
+ # in exactly one place — it was MOVED out of the head, never duplicated, so
21
+ # a turn pays for it once.
11
22
  class Briefing < ContextProvider
12
23
  def initialize(session_store:)
13
24
  @session_store = session_store
@@ -34,12 +45,10 @@ module Insika
34
45
  declared = Array(request.profile.briefing_fields)
35
46
  return [] if declared.empty? # defensive; enabled_for? already gates
36
47
 
37
- block = format_block(declared, fields, briefing["next_step"])
38
- return [] if block.nil?
48
+ missing = declared.reject { |name| Coercion.present?(fields[name]) }
49
+ next_step = briefing["next_step"]
39
50
 
40
- [ContextFragment.build(content: block, placement: :system,
41
- priority: Context::Priority::BRIEFING,
42
- source: id)]
51
+ [head_fragment(declared, fields), tail_fragment(missing, next_step)].compact
43
52
  end
44
53
 
45
54
  private
@@ -52,37 +61,57 @@ module Insika
52
61
  @session_store.find(session.id)&.briefing || {}
53
62
  end
54
63
 
55
- # Byte contract (the specs assert this shape):
64
+ # The HEAD block — durable facts only. Byte contract:
56
65
  # <briefing>
57
66
  # known:
58
67
  # size: M
59
- # still missing: delivery_day
60
- # next step: send the payment link tomorrow at 10
61
68
  # </briefing>
62
- # Rules: `known` renders only when at least one declared field has a
63
- # stored value; `still missing` renders every declared field with no
64
- # stored value (including the all-missing case — that is the block's
65
- # job); `next step` renders only when non-nil; stored keys NOT in the
66
- # declaration are never rendered (they stay in the store and reappear if
67
- # the pack re-declares them).
68
- def format_block(declared, fields, next_step)
69
+ # Nothing known yet -> no fragment at all (an empty `known:` header
70
+ # teaches the model nothing and still costs a cache invalidation).
71
+ # Stored keys NOT in the declaration are never rendered (they stay in the
72
+ # store and reappear if the pack re-declares them).
73
+ def head_fragment(declared, fields)
69
74
  known = declared.filter_map do |name|
70
75
  " #{name}: #{flatten(fields[name])}" if Coercion.present?(fields[name])
71
76
  end
72
- missing = declared.reject { |name| Coercion.present?(fields[name]) }
77
+ return nil if known.empty?
73
78
 
79
+ block = <<~BLOCK.strip
80
+ <briefing>
81
+ known:
82
+ #{known.join("\n")}
83
+ </briefing>
84
+ BLOCK
85
+ ContextFragment.build(content: block, placement: :system,
86
+ priority: Context::Priority::BRIEFING, source: id)
87
+ end
88
+
89
+ # The TAIL recitation — two lines, no more. Byte contract:
90
+ # <recitation>
91
+ # still missing: delivery_day
92
+ # next step: send the payment link tomorrow at 10
93
+ # </recitation>
94
+ # `still missing` renders every declared field with no stored value
95
+ # (including the all-missing case — that is this block's job); `next step`
96
+ # renders only when non-nil. Neither -> no fragment.
97
+ #
98
+ # A `user` message, like every other engine append inside a turn
99
+ # (LoopDetector, TurnBudget): the system prefix stays byte-stable, so the
100
+ # cache breakpoint at its end keeps hitting.
101
+ def tail_fragment(missing, next_step)
74
102
  lines = []
75
- lines << "known:" unless known.empty?
76
- lines.concat(known)
77
103
  lines << "still missing: #{missing.join(', ')}" unless missing.empty?
78
104
  lines << "next step: #{flatten(next_step)}" if Coercion.present?(next_step)
79
105
  return nil if lines.empty?
80
106
 
81
- <<~BLOCK.strip
82
- <briefing>
107
+ block = <<~BLOCK.strip
108
+ <recitation>
83
109
  #{lines.join("\n")}
84
- </briefing>
110
+ </recitation>
85
111
  BLOCK
112
+ ContextFragment.build(content: { role: :user, content: block },
113
+ placement: :tail,
114
+ priority: Context::Priority::RECITATION, source: id)
86
115
  end
87
116
 
88
117
  # utf8 the value and flatten newlines/whitespace so a value can never
@@ -0,0 +1,108 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ module Context
5
+ module Providers
6
+ # Level 1 (progressive disclosure) of what the engine has LEARNED, as
7
+ # opposed to what a human curated (Skill) or the model wrote mid-turn
8
+ # (Memory). Retrieval is per-message and dynamic, so — like
9
+ # `SkillTrigger`, unlike the static `CatalogProvider` subclasses — this
10
+ # builds its `<knowledge>` block directly in `call`, never a fixed list.
11
+ class Knowledge < ContextProvider
12
+ def initialize(store:)
13
+ @store = store
14
+ # One Index PER TYPE, built once and reused for every agent/turn —
15
+ # never per call. This provider instance itself lives for the
16
+ # process's lifetime (built once at boot, see wiring), so an
17
+ # Index rebuilt fresh each call would throw away its own read
18
+ # cache (Index::Scan's dominant cost is re-parsing YAML
19
+ # frontmatter; measured, not assumed) on every single turn.
20
+ # Keyed by the config's `index` string so a future FTS5 agent
21
+ # gets its own instance, never Scan's.
22
+ @indexes = Hash.new { |h, index_name| h[index_name] = Insika::Knowledge::Index.build({ "index" => index_name }, store: @store) }
23
+ end
24
+
25
+ # Per-agent opt-in (`knowledge.retrieve`), like Memory's `profile.memory`.
26
+ def enabled_for?(profile) = !!(profile.knowledge && Coercion.truthy?(profile.knowledge["retrieve"]))
27
+
28
+ # required? == false (default): a store failure degrades to a
29
+ # :provider_warning, never aborts the turn.
30
+ def call(request)
31
+ config = request.profile.knowledge
32
+ return [] unless config
33
+
34
+ top_k = positive_int(config["top_k"]) || 5
35
+ index = @indexes[config["index"].to_s]
36
+ matches = index.search(request.profile.id, tenant: request.tenant,
37
+ query: request.message.to_s, top_k: top_k)
38
+ return [] if matches.empty?
39
+
40
+ hits = matches.map { |c| [c, "top-K match"] } +
41
+ expand_links(matches, request, top_k).map { |c| [c, "one-hop link"] }
42
+
43
+ [ContextFragment.build(
44
+ content: format_block(hits), placement: :system,
45
+ priority: Context::Priority::KNOWLEDGE, source: id,
46
+ labels: hits.map { |c, reason| { "name" => c[:name], "reason" => reason } }
47
+ )]
48
+ end
49
+
50
+ private
51
+
52
+ def positive_int(value)
53
+ n = value.to_i
54
+ n.positive? ? n : nil
55
+ end
56
+
57
+ # ONE level, deliberately — the same "cannot work without" reasoning
58
+ # `SkillTrigger#companions` applies to a skill's declared companions:
59
+ # a transitive walk would make a cycle a hang and a chain a budget
60
+ # blowout. Newly-discovered concepts (not already in the top-K) are
61
+ # capped at top_k again — "that one hop is the whole graph benefit at
62
+ # ~1% of the graph cost", not a second unbounded retrieval.
63
+ def expand_links(matches, request, top_k)
64
+ known = matches.map { |c| c[:name] }
65
+ discovered = []
66
+ matches.each do |concept|
67
+ Insika::Knowledge::Concept.links(concept[:body]).each do |name|
68
+ next if known.include?(name) || discovered.any? { |d| d[:name] == name }
69
+
70
+ found = fetch(request, name)
71
+ discovered << found if found
72
+ end
73
+ end
74
+ discovered.first(top_k)
75
+ end
76
+
77
+ def fetch(request, name)
78
+ raw = @store.get(request.profile.id, name, tenant: request.tenant)
79
+ raw && Insika::Knowledge::Concept.parse(raw)
80
+ end
81
+
82
+ # Level 1 only — name/description/confidence/provenance, never the
83
+ # body (that is `load_knowledge`'s job). The instruction is the exact
84
+ # lesson the knowledge-adoption experiment drew: a polite "when to
85
+ # use" scored near zero; an explicit, ordered rule naming the tool
86
+ # held up. Present only when there is something to point at.
87
+ def format_block(hits)
88
+ entries = hits.map do |c, _reason|
89
+ %( <concept name="#{c[:name]}" confidence="#{format('%.2f', c[:confidence])}" ) +
90
+ %(provenance="#{c[:provenance]}">#{c[:description]}</concept>)
91
+ end.join("\n")
92
+
93
+ <<~BLOCK.strip
94
+ <knowledge>
95
+ #{entries}
96
+ </knowledge>
97
+
98
+ If the customer's question needs more than the summary above, call
99
+ `load_knowledge("name")` FIRST — before any other lookup for that
100
+ topic. This is learned from past conversations, not official
101
+ policy: never state a `provenance="observed"` concept to the
102
+ customer as a guarantee.
103
+ BLOCK
104
+ end
105
+ end
106
+ end
107
+ end
108
+ end
@@ -8,12 +8,16 @@ module Insika
8
8
  # a WRONG agent, not a degraded one. prompt_refs: priority 90 pinned
9
9
  # fragments, from the PromptCatalog (catalog defaults to nil).
10
10
  #
11
- # PER-AGENT identity. `profile.prompt_files` (file names)
12
- # wins over the wiring's `files:` — this fixes the limitation of a new
13
- # agent inheriting the previous persona's prompt. The content comes from `agent_files`
14
- # (AgentFileStore, lives in the Store), with a File.read fallback
15
- # for on-disk paths (compat/seed). Without prompt_files -> uses the wiring's
16
- # `files:` (deployment default; byte-for-byte parity).
11
+ # PER-AGENT identity, from `profile.prompt_files`/`base_prompt` only — an
12
+ # agent that declares neither has no identity here, and there is no
13
+ # deployment-wide fallback file to borrow one from. A shared fallback
14
+ # (a prior design: an agent without prompt_files silently inherited a
15
+ # wiring-level default) is exactly how a real deployment answered as the
16
+ # WRONG business: a `copilot` data agent provisioned without its own
17
+ # identity inherited the deployment's demo persona ("Bia, Pizzaria do
18
+ # Zé") byte for byte, confirmed live. An agent never wears another
19
+ # agent's identity — `@base`/`system_files` stay legitimate (the same
20
+ # generic content for every agent, never one agent's specific persona).
17
21
  class Prompt < ContextProvider
18
22
  # Engine-owned execution discipline, appended AFTER the agent's identity.
19
23
  # The one behavior every reference harness bakes into its base prompt
@@ -28,11 +32,12 @@ module Insika
28
32
  "query, use a synonym or broader term, drop a secondary filter — before telling " \
29
33
  "the user you found nothing. Do not narrate the retries. Then conclude.\n" \
30
34
  "- Tool error: read the error, fix the arguments or try another path; never " \
31
- "repeat the exact same call."
35
+ "repeat the exact same call.\n" \
36
+ "- A URL in a tool result (e.g. a `url` field): quote it byte-for-byte. Never " \
37
+ "construct, guess, or rewrite the domain, host, or path."
32
38
 
33
- def initialize(base: "", files: [], catalog: nil, agent_files: nil, system_files: nil)
39
+ def initialize(base: "", catalog: nil, agent_files: nil, system_files: nil)
34
40
  @base = base
35
- @files = Array(files)
36
41
  @catalog = catalog
37
42
  @agent_files = agent_files
38
43
  @system_files = system_files
@@ -43,12 +48,18 @@ module Insika
43
48
  def layer = :identity
44
49
 
45
50
  def call(request)
46
- fragments = []
47
51
  identity = build_identity(request.profile)
48
- unless identity.empty?
49
- fragments << ContextFragment.build(content: identity, placement: :system,
50
- priority: Context::Priority::IDENTITY, source: id, pinned: true)
52
+ if identity.empty?
53
+ raise ContextError.new(
54
+ "agent '#{request.profile&.id}' has no identity of its own (no base_prompt, " \
55
+ "no prompt_files) — refusing to run rather than answer with no identity or " \
56
+ "another agent's",
57
+ provider: id
58
+ )
51
59
  end
60
+
61
+ fragments = [ContextFragment.build(content: identity, placement: :system,
62
+ priority: Context::Priority::IDENTITY, source: id, pinned: true)]
52
63
  fragments.concat(ref_fragments(request.profile))
53
64
  fragments
54
65
  end
@@ -59,10 +70,9 @@ module Insika
59
70
  # A SINGLE fragment preserves the internal base->files order (sorting
60
71
  # acts only BETWEEN fragments) and guarantees byte-for-byte parity.
61
72
  #
62
- # profile.prompt_files (names) wins over @files (wiring): an agent with
63
- # its own identity does not inherit the deployment's. Each source resolves
64
- # via AgentFileStore (per agent) OR File.read (on-disk path) — in that
65
- # order. Without prompt_files, falls back to the wiring's @files.
73
+ # profile.prompt_files: each source resolves via AgentFileStore (per
74
+ # agent) OR File.read (on-disk path) — in that order. No other agent's
75
+ # files are ever read for this one; there is no wiring-level fallback.
66
76
  def build_identity(profile)
67
77
  parts = [@base]
68
78
  parts.concat(system_parts) # global system files, for EVERY agent
@@ -73,15 +83,11 @@ module Insika
73
83
  # NO identity at all, and a chatty model answered plausibly enough to hide
74
84
  # it. Additive to prompt_files, not exclusive: an agent may carry both.
75
85
  parts << profile&.base_prompt.to_s
76
- sources = Array(profile&.prompt_files)
77
- if sources.empty?
78
- @files.each { |f| parts << File.read(f, encoding: "UTF-8") if File.exist?(f) }
79
- else
80
- sources.each { |src| parts << read_source(profile&.id, src.to_s) }
81
- end
86
+ Array(profile&.prompt_files).each { |src| parts << read_source(profile&.id, src.to_s) }
82
87
  identity = parts.reject { |p| p.nil? || p.strip.empty? }.join("\n\n")
83
88
  # Discipline rides an EXISTING identity, never substitutes one: an
84
- # agent with no identity at all must stay detectably empty.
89
+ # empty identity here is fatal (`call` raises), not silently patched
90
+ # with the engine's own boilerplate.
85
91
  return identity if identity.empty? || !tool_persistence?(profile)
86
92
 
87
93
  "#{identity}\n\n#{TOOL_PERSISTENCE}"