insika 0.3.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +296 -0
- data/README.md +48 -12
- data/bin/insika +725 -0
- data/bin/insika-router +87 -0
- data/docs/AGENTS.md +116 -406
- data/docs/API.md +5 -5
- data/docs/ARCHITECTURE.md +3 -2
- data/docs/ARTIFACTS.md +137 -0
- data/docs/BENCHMARK.md +2 -2
- data/docs/CHANNELS.md +14 -14
- data/docs/CONTEXT.md +63 -19
- data/docs/DEMO.md +80 -0
- data/docs/DEPLOY.md +87 -10
- data/docs/EMBEDDING.md +1 -1
- data/docs/EVALS.md +128 -3
- data/docs/FACTS.md +3 -3
- data/docs/HARVEST.md +5 -6
- data/docs/KNOWLEDGE.md +290 -0
- data/docs/LOADTEST.md +17 -29
- data/docs/MEDIA.md +128 -0
- data/docs/OBSERVABILITY.md +46 -12
- data/docs/OUTCOMES.md +137 -0
- data/docs/PLUGINS.md +51 -6
- data/docs/POLICY.md +222 -0
- data/docs/REFINEMENT.md +14 -9
- data/docs/RELEASING.md +4 -4
- data/docs/ROUTER.md +213 -0
- data/docs/RUNNING-LOCAL.md +5 -5
- data/docs/SCHEDULING.md +121 -0
- data/docs/SECURITY.md +23 -7
- data/docs/SKILLS.md +11 -2
- data/docs/SOAK.md +3 -3
- data/docs/TEMPLATES.md +134 -0
- data/docs/TOOLS.md +176 -27
- data/docs/WHY.md +1 -1
- data/docs/WORKFLOWS.md +2 -2
- data/docs/_includes/head_custom.html +5 -0
- data/docs/_includes/title.html +13 -0
- data/docs/_sass/color_schemes/insika.scss +32 -0
- data/docs/_sass/custom/custom.scss +199 -0
- data/docs/_sass/custom/setup.scss +26 -0
- data/docs/assets/img/favicon.svg +7 -0
- data/docs/assets/img/insika-mark.svg +7 -0
- data/docs/core-concepts.md +21 -0
- data/docs/domain.md +4 -4
- data/docs/improve.md +20 -0
- data/docs/index.md +8 -5
- data/docs/integrate.md +20 -0
- data/docs/operate.md +13 -6
- data/docs/prompts/ADD-TOOL.md +118 -0
- data/docs/prompts/DIAGNOSE-TURN.md +65 -0
- data/docs/prompts/GO-LIVE.md +138 -0
- data/docs/prompts/RUN-EXAMPLES.md +70 -0
- data/docs/reference.md +19 -0
- data/docs/ship.md +10 -2
- data/docs/start-here.md +18 -0
- data/lib/insika/agent_profile.rb +99 -17
- data/lib/insika/artifact_signing.rb +82 -0
- data/lib/insika/artifact_store.rb +160 -0
- data/lib/insika/channel_delivery.rb +1 -1
- data/lib/insika/chat_builder.rb +50 -19
- data/lib/insika/commands/agent_payload.rb +2 -2
- data/lib/insika/commands/backfill_knowledge.rb +145 -0
- data/lib/insika/commands/delete_artifact.rb +35 -0
- data/lib/insika/commands/delete_concept.rb +34 -0
- data/lib/insika/commands/delete_mcp.rb +6 -2
- data/lib/insika/commands/delete_tenant_data.rb +15 -3
- data/lib/insika/commands/gate_refinement.rb +1 -1
- data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
- data/lib/insika/commands/restore_concept.rb +34 -0
- data/lib/insika/commands/seed_demo_data.rb +31 -0
- data/lib/insika/commands/upsert_mcp.rb +6 -3
- data/lib/insika/commands/write_concept.rb +57 -0
- data/lib/insika/compaction.rb +196 -0
- data/lib/insika/context/builder.rb +6 -2
- data/lib/insika/context/fragment.rb +4 -1
- data/lib/insika/context/priority.rb +8 -0
- data/lib/insika/context/providers/briefing.rb +53 -24
- data/lib/insika/context/providers/knowledge.rb +108 -0
- data/lib/insika/context/providers/prompt.rb +30 -24
- data/lib/insika/context/providers/session.rb +46 -10
- data/lib/insika/context_trace_store.rb +11 -1
- data/lib/insika/cron.rb +189 -0
- data/lib/insika/demo/agent_attrs.rb +43 -0
- data/lib/insika/demo/golden_cases.rb +81 -0
- data/lib/insika/demo/seeder.rb +336 -0
- data/lib/insika/doctor.rb +280 -17
- data/lib/insika/dsl/definition.rb +3 -2
- data/lib/insika/dsl/runtime.rb +64 -79
- data/lib/insika/dsl/server_boot.rb +23 -1
- data/lib/insika/dsl/system.rb +10 -2
- data/lib/insika/dsl.rb +103 -2
- data/lib/insika/env_schema.rb +21 -7
- data/lib/insika/evals/golden.rb +41 -4
- data/lib/insika/evals/judge.rb +47 -2
- data/lib/insika/evals/pairwise.rb +11 -0
- data/lib/insika/evals/persona.rb +98 -0
- data/lib/insika/evals/runner.rb +9 -0
- data/lib/insika/evals/simulator.rb +225 -0
- data/lib/insika/evals/transport.rb +84 -2
- data/lib/insika/event_stream.rb +10 -0
- data/lib/insika/executor.rb +295 -55
- data/lib/insika/followup_policy.rb +2 -25
- data/lib/insika/golden_store.rb +16 -1
- data/lib/insika/grounding/matcher.rb +1 -1
- data/lib/insika/knowledge.rb +680 -0
- data/lib/insika/knowledge_store.rb +140 -0
- data/lib/insika/loop_detector.rb +5 -34
- data/lib/insika/mcp_client.rb +94 -0
- data/lib/insika/mcp_json.rb +74 -0
- data/lib/insika/mcp_live_tool.rb +43 -0
- data/lib/insika/mcp_store.rb +98 -26
- data/lib/insika/mcp_tool_ingestor.rb +30 -8
- data/lib/insika/mcp_tool_registry.rb +100 -0
- data/lib/insika/media.rb +115 -31
- data/lib/insika/message_origin.rb +1 -1
- data/lib/insika/middleware.rb +9 -0
- data/lib/insika/onboarding.rb +17 -1
- data/lib/insika/outcome_store.rb +1 -1
- data/lib/insika/overlay_tool_registry.rb +37 -17
- data/lib/insika/packaging.rb +2 -2
- data/lib/insika/profile_source.rb +15 -1
- data/lib/insika/prompt_catalog.rb +10 -0
- data/lib/insika/retention.rb +36 -1
- data/lib/insika/router/app.rb +157 -0
- data/lib/insika/router/backend_pool.rb +98 -0
- data/lib/insika/router/hash_ring.rb +55 -0
- data/lib/insika/router/proxy_body.rb +34 -0
- data/lib/insika/router/session_key.rb +54 -0
- data/lib/insika/router.rb +18 -0
- data/lib/insika/schedule.rb +177 -0
- data/lib/insika/schedule_engine.rb +314 -0
- data/lib/insika/schedule_store.rb +208 -0
- data/lib/insika/server/app.rb +105 -15
- data/lib/insika/server/rack_app.rb +5 -1
- data/lib/insika/server/responses.rb +5 -5
- data/lib/insika/session_store.rb +34 -4
- data/lib/insika/settings_store.rb +8 -1
- data/lib/insika/skill_catalog.rb +12 -0
- data/lib/insika/soak/runner.rb +4 -4
- data/lib/insika/steer_injector.rb +21 -10
- data/lib/insika/studio/app.rb +591 -47
- data/lib/insika/studio/assets/dist/application.css +1 -1
- data/lib/insika/studio/assets/dist/application.js +21 -21
- data/lib/insika/studio/forms.rb +57 -5
- data/lib/insika/studio/nav_icons.rb +14 -1
- data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
- data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
- data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
- data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
- data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
- data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
- data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
- data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
- data/lib/insika/studio/views/_agents_master.erb +44 -0
- data/lib/insika/studio/views/_message.erb +49 -32
- data/lib/insika/studio/views/agent_detail.erb +61 -820
- data/lib/insika/studio/views/agents.erb +70 -57
- data/lib/insika/studio/views/artifact.erb +23 -0
- data/lib/insika/studio/views/artifacts.erb +59 -0
- data/lib/insika/studio/views/evals.erb +2 -2
- data/lib/insika/studio/views/facts.erb +1 -1
- data/lib/insika/studio/views/funnel.erb +1 -1
- data/lib/insika/studio/views/home.erb +106 -67
- data/lib/insika/studio/views/knowledge.erb +123 -0
- data/lib/insika/studio/views/layout.erb +14 -11
- data/lib/insika/studio/views/mcp.erb +174 -80
- data/lib/insika/studio/views/session.erb +231 -177
- data/lib/insika/studio/views/settings.erb +50 -1
- data/lib/insika/studio/views/skills.erb +1 -1
- data/lib/insika/studio/views/tools.erb +24 -9
- data/lib/insika/telemetry/recorder.rb +49 -1
- data/lib/insika/templates/browser-agent/README.md +36 -0
- data/lib/insika/templates/browser-agent/agent.rb +49 -0
- data/lib/insika/templates/daily-digest/README.md +47 -0
- data/lib/insika/templates/daily-digest/agent.rb +77 -0
- data/lib/insika/templates/repo-explorer/README.md +36 -0
- data/lib/insika/templates/repo-explorer/agent.rb +45 -0
- data/lib/insika/templates/research-analyst/README.md +26 -0
- data/lib/insika/templates/research-analyst/agent.rb +68 -0
- data/lib/insika/templates/review-panel/README.md +20 -0
- data/lib/insika/templates/review-panel/agent.rb +50 -0
- data/lib/insika/templates/travel-planner/README.md +35 -0
- data/lib/insika/templates/travel-planner/agent.rb +87 -0
- data/lib/insika/templates.rb +112 -0
- data/lib/insika/tick.rb +24 -12
- data/lib/insika/timezone.rb +45 -0
- data/lib/insika/tool_batch.rb +67 -0
- data/lib/insika/tool_usage_report.rb +162 -0
- data/lib/insika/tools/generate_image.rb +52 -7
- data/lib/insika/tools/load_knowledge.rb +74 -0
- data/lib/insika/tools/run_persona_eval.rb +328 -0
- data/lib/insika/tools/save_artifact.rb +95 -0
- data/lib/insika/turn_budget.rb +91 -0
- data/lib/insika/turn_output.rb +1 -1
- data/lib/insika/turn_state.rb +15 -4
- data/lib/insika/version.rb +1 -1
- data/lib/insika/wiring/graph.rb +184 -12
- data/lib/insika/wiring/graph_chat.rb +102 -0
- data/lib/insika.rb +64 -0
- metadata +109 -5
- data/docs/build.md +0 -14
- data/docs/understand.md +0 -10
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Insika
|
|
4
|
+
# In-session compaction: when a session's UNCOMPACTED transcript
|
|
5
|
+
# grows past `compact_after` messages, everything but the last `keep_last`
|
|
6
|
+
# is summarized by a cheap model into one fragment; the tail stays verbatim.
|
|
7
|
+
# This module is the pure half — boundary math, the prompt, and the
|
|
8
|
+
# Summarizer over an injected ask (the Distiller shape: unit-testable
|
|
9
|
+
# without a provider). The trigger lives in the Executor (post-turn, off
|
|
10
|
+
# the critical path); the persistence in SessionStore#set_compaction; the
|
|
11
|
+
# read path in Context::Providers::Session.
|
|
12
|
+
module Compaction
|
|
13
|
+
DEFAULT_KEEP_LAST = 20
|
|
14
|
+
DEFAULT_COMPACT_AFTER = 40
|
|
15
|
+
# A summary that outgrows this is truncated — the compaction must never
|
|
16
|
+
# grow the context it exists to shrink.
|
|
17
|
+
MAX_SUMMARY_CHARS = 6_000
|
|
18
|
+
# Per-message cap in the transcript slice sent to the summarizer (a
|
|
19
|
+
# `role: tool` body can be 4 000 chars in the store); the head of a long
|
|
20
|
+
# result carries the identity of what happened, which is what a summary needs.
|
|
21
|
+
MESSAGE_CHAR_CAP = 1_000
|
|
22
|
+
|
|
23
|
+
# The engine's generic prompt. A platform `compaction.prompt` REPLACES it
|
|
24
|
+
# wholesale (the distill convention) — the engine never writes store
|
|
25
|
+
# vocabulary. The preserve-list is the P28 contract: facts (CEP, order
|
|
26
|
+
# numbers), commitments, the MISSING list, decisions.
|
|
27
|
+
DEFAULT_PROMPT = <<~PROMPT.freeze
|
|
28
|
+
You are compacting the OLD part of an ongoing customer conversation into
|
|
29
|
+
one summary that the assistant will read INSTEAD of those messages. The
|
|
30
|
+
recent messages stay verbatim; your summary is the only surviving trace
|
|
31
|
+
of the old ones — anything you drop is gone for good.
|
|
32
|
+
|
|
33
|
+
Preserve, verbatim where short:
|
|
34
|
+
- every fact the customer stated (sizes, budget, address, postal code/CEP,
|
|
35
|
+
order numbers, product choices, dates, quantities);
|
|
36
|
+
- every commitment the assistant made (promises, prices quoted, delivery
|
|
37
|
+
windows, agreed next steps);
|
|
38
|
+
- what was asked and is still unanswered (the missing information);
|
|
39
|
+
- decisions already made, so nothing gets re-asked or re-litigated.
|
|
40
|
+
|
|
41
|
+
Do not invent, do not editorialize, do not add advice. Answer with the
|
|
42
|
+
summary text only — plain text, compact, in the conversation's own language.
|
|
43
|
+
PROMPT
|
|
44
|
+
|
|
45
|
+
# The compaction plan: summarize messages[from...upto] (from = the previous
|
|
46
|
+
# boundary), keep messages[upto..] verbatim. count = upto - from.
|
|
47
|
+
Plan = Data.define(:from, :upto, :count)
|
|
48
|
+
|
|
49
|
+
module_function
|
|
50
|
+
|
|
51
|
+
# Decides whether (and what) to compact. -> Plan | nil.
|
|
52
|
+
# messages: the session transcript (append-only, RFC-0016).
|
|
53
|
+
# state: the persisted "compaction" hash ({"upto"=>, ...}) | nil.
|
|
54
|
+
# config: the Settings "compaction" hash (keep_last/compact_after).
|
|
55
|
+
# `compact_after` is clamped to at least `keep_last` so the plan always
|
|
56
|
+
# moves the boundary forward. The boundary retreats over `role: "tool"`
|
|
57
|
+
# messages so an eviction unit (assistant-with-tool_calls + its results)
|
|
58
|
+
# is never split — the whole cycle stays verbatim instead.
|
|
59
|
+
def plan(messages:, state:, config:)
|
|
60
|
+
msgs = Array(messages)
|
|
61
|
+
keep_last = positive(config && config["keep_last"], DEFAULT_KEEP_LAST)
|
|
62
|
+
compact_after = positive(config && config["compact_after"], DEFAULT_COMPACT_AFTER)
|
|
63
|
+
compact_after = keep_last if compact_after < keep_last
|
|
64
|
+
from = state ? state["upto"].to_i : 0
|
|
65
|
+
return nil unless msgs.size - from > compact_after
|
|
66
|
+
|
|
67
|
+
upto = msgs.size - keep_last
|
|
68
|
+
upto -= 1 while upto > from && role_of(msgs[upto]) == "tool"
|
|
69
|
+
return nil unless upto > from
|
|
70
|
+
|
|
71
|
+
Plan.new(from: from, upto: upto, count: upto - from)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The full prompt for one compaction run: the base rules, the PREVIOUS
|
|
75
|
+
# summary (so a fact from turn 3 survives every re-compaction — each
|
|
76
|
+
# summary folds the last one in) and only the NEW slice. The slice is
|
|
77
|
+
# sent UNREDACTED on purpose: it replaces transcript the main model
|
|
78
|
+
# already reads raw, inside the same trust boundary, and redaction would
|
|
79
|
+
# delete exactly the facts (CEP, order id) the summary must preserve.
|
|
80
|
+
def prompt(messages:, plan:, previous: nil, base: nil)
|
|
81
|
+
rules = Coercion.presence(base.to_s) || DEFAULT_PROMPT
|
|
82
|
+
parts = [rules.rstrip]
|
|
83
|
+
if Coercion.presence(previous.to_s)
|
|
84
|
+
parts << "## The summary so far (fold it into the new one — its facts must survive)\n\n#{previous}"
|
|
85
|
+
end
|
|
86
|
+
parts << "## The messages to compact\n\n#{transcript(messages, plan)}"
|
|
87
|
+
parts.join("\n\n")
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# "[i] role: content" over the plan's slice, one line per message; a
|
|
91
|
+
# tool-calling assistant message with no text renders the tool names.
|
|
92
|
+
def transcript(messages, plan)
|
|
93
|
+
Array(messages)[plan.from...plan.upto].to_a.each_with_index.map do |msg, offset|
|
|
94
|
+
"[#{plan.from + offset}] #{role_of(msg)}: #{text_of(msg)}"
|
|
95
|
+
end.join("\n")
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def role_of(msg) = (msg["role"] || msg[:role]).to_s
|
|
99
|
+
|
|
100
|
+
def text_of(msg)
|
|
101
|
+
content = (msg["content"] || msg[:content]).to_s.strip.gsub(/\s+/, " ")
|
|
102
|
+
if content.empty?
|
|
103
|
+
calls = msg["tool_calls"] || msg[:tool_calls]
|
|
104
|
+
names = Array(calls).filter_map { |c| c.is_a?(Hash) ? (c["name"] || c[:name] || c.dig("function", "name")) : nil }
|
|
105
|
+
content = names.empty? ? "(empty)" : "(tool calls: #{names.join(', ')})"
|
|
106
|
+
end
|
|
107
|
+
content[0, MESSAGE_CHAR_CAP]
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def positive(value, default)
|
|
111
|
+
n = value.to_i
|
|
112
|
+
n.positive? ? n : default
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# The one place compaction asks a model for anything. Pure over an
|
|
116
|
+
# injected `ask` (the Distiller shape); the real ask is a lambda built by
|
|
117
|
+
# SummarizerFactory (ruby_llm required lazily, load_guard stays green).
|
|
118
|
+
class Summarizer
|
|
119
|
+
# A blank answer must not overwrite the boundary — empty output is a
|
|
120
|
+
# loud failure, never "the old turns said nothing".
|
|
121
|
+
class Unusable < Insika::ValidationError; end
|
|
122
|
+
|
|
123
|
+
# ask: ->(prompt) { "<raw model text>" } | something answering #content
|
|
124
|
+
# (+ #input_tokens/#output_tokens/#cached_tokens for cost).
|
|
125
|
+
# model: the ref recorded on the event ("utility_model" default).
|
|
126
|
+
attr_reader :model
|
|
127
|
+
|
|
128
|
+
def initialize(ask:, model: "utility_model")
|
|
129
|
+
@ask = ask
|
|
130
|
+
@model = model.to_s
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# -> { summary: String, cost: { "spent" => N, "cached" => N } | nil }
|
|
134
|
+
# Raises Unusable on a blank answer; truncates past MAX_SUMMARY_CHARS.
|
|
135
|
+
def summarize(prompt:)
|
|
136
|
+
answer = @ask.call(prompt)
|
|
137
|
+
text = Coercion.utf8(text_of(answer)).strip
|
|
138
|
+
raise Unusable, "the summarizer answered with nothing" if text.empty?
|
|
139
|
+
|
|
140
|
+
{ summary: text[0, MAX_SUMMARY_CHARS], cost: cost_of(answer) }
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
private
|
|
144
|
+
|
|
145
|
+
def text_of(answer) = (answer.respond_to?(:content) ? answer.content : answer).to_s
|
|
146
|
+
|
|
147
|
+
# nil when the provider said nothing — never 0 (the Distiller's
|
|
148
|
+
# discipline). The cached prefix is INCLUDED in the spent total.
|
|
149
|
+
def cost_of(answer)
|
|
150
|
+
return nil unless answer.respond_to?(:input_tokens) && answer.respond_to?(:output_tokens)
|
|
151
|
+
|
|
152
|
+
input = answer.input_tokens.to_i
|
|
153
|
+
output = answer.output_tokens.to_i
|
|
154
|
+
cached = answer.respond_to?(:cached_tokens) ? answer.cached_tokens.to_i : 0
|
|
155
|
+
{ "spent" => input + output, "cached" => cached }
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Resolves WHICH model summarizes and builds the ask. compaction.model ->
|
|
160
|
+
# platform utility_model -> nil (nil means "feature inert", never a guess —
|
|
161
|
+
# the DistillerFactory ladder). NOT the fallbacks chain: that one answers
|
|
162
|
+
# "which model serves the customer turn". `ask_factory`/`llm` injectable (specs).
|
|
163
|
+
module SummarizerFactory
|
|
164
|
+
module_function
|
|
165
|
+
|
|
166
|
+
# config: the Settings "compaction" hash. -> Summarizer | nil
|
|
167
|
+
def build(config, utility_model: nil, ask_factory: nil, llm: nil)
|
|
168
|
+
ref = Coercion.presence(config && config["model"]) || Coercion.presence(utility_model)
|
|
169
|
+
return nil if ref.nil?
|
|
170
|
+
|
|
171
|
+
provider, model = split_ref(ref)
|
|
172
|
+
factory = ask_factory || ->(m, p) { ruby_llm_ask(m, p, llm: llm) }
|
|
173
|
+
Summarizer.new(ask: factory.call(model, provider), model: ref)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# "provider/model" -> [provider, model]; "model" -> [nil, model] — one
|
|
177
|
+
# syntax for "which model" across features (DistillerFactory's reading).
|
|
178
|
+
def split_ref(ref)
|
|
179
|
+
prov, name = ref.to_s.split("/", 2)
|
|
180
|
+
name ? [prov, name] : [nil, prov]
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Temperature 0: the same slice must compact to the same boundary
|
|
184
|
+
# deterministically. `ruby_llm` is required lazily so nothing loads a
|
|
185
|
+
# provider gem until compaction is actually configured (load_guard stays green).
|
|
186
|
+
def ruby_llm_ask(model, provider, llm: nil)
|
|
187
|
+
require "ruby_llm"
|
|
188
|
+
llm ||= RubyLLM
|
|
189
|
+
lambda do |prompt|
|
|
190
|
+
llm.chat(model: model, provider: provider, assume_model_exists: true)
|
|
191
|
+
.with_temperature(0).ask(prompt)
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
end
|
|
@@ -151,13 +151,17 @@ module Insika
|
|
|
151
151
|
identity, volatile = system_frags.partition { |f| (f.layer || :volatile) == :identity }
|
|
152
152
|
system_frags = sort_canonical(identity) + sort_canonical(volatile)
|
|
153
153
|
history_frags = fragments.select { |f| f.placement == :history } # production order (chronological)
|
|
154
|
+
# :tail renders after ALL history — the last thing before the current user
|
|
155
|
+
# message. Attention is strongest at the end of the context, so the goal
|
|
156
|
+
# restated here survives a 30-call turn that the head prompt no longer does.
|
|
157
|
+
tail_frags = fragments.select { |f| f.placement == :tail }
|
|
154
158
|
tool_frags = fragments.select { |f| f.placement == :tool_context }
|
|
155
159
|
|
|
156
160
|
system = system_frags.map(&:content).join("\n\n")
|
|
157
|
-
history = history_frags.map(&:content)
|
|
161
|
+
history = history_frags.map(&:content) + tail_frags.map(&:content)
|
|
158
162
|
tool_context = tool_frags.empty? ? nil : tool_frags.map(&:content).join("\n\n")
|
|
159
163
|
|
|
160
|
-
canonical = system_frags + history_frags + tool_frags
|
|
164
|
+
canonical = system_frags + history_frags + tail_frags + tool_frags
|
|
161
165
|
ContextPackage.new(
|
|
162
166
|
system: system, history: history, tool_context: tool_context,
|
|
163
167
|
fragments: canonical, budget: { cap: cap, used: canonical.sum(&:tokens), evicted: evicted }
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
module Insika
|
|
4
4
|
# Unit of context produced by a provider.
|
|
5
5
|
# SHARED type (Insika::, not Insika::Context::).
|
|
6
|
-
# placement: :system | :history | :tool_context
|
|
6
|
+
# placement: :system | :history | :tail | :tool_context
|
|
7
|
+
# :tail renders AFTER the whole history, i.e. as the last thing
|
|
8
|
+
# the model reads before the current user message. Its content is
|
|
9
|
+
# a message Hash ({role:, content:}), like a :history fragment.
|
|
7
10
|
# priority: Integer; higher = more important (survives cuts)
|
|
8
11
|
# tokens: Integer | nil; estimated by the Builder when nil
|
|
9
12
|
# source: String — provider id (audit)
|
|
@@ -17,9 +17,14 @@ module Insika
|
|
|
17
17
|
# the injected block.
|
|
18
18
|
module Priority
|
|
19
19
|
IDENTITY = 100 # IDENTITY/SOUL (Prompt) — pinned
|
|
20
|
+
RECITATION = 95 # <recitation> the goal restated at the TAIL of context
|
|
21
|
+
# (Briefing). Never cut: it is two lines, and the one
|
|
22
|
+
# turn it gets evicted is the long turn that needed it.
|
|
20
23
|
PROMPT_REF = 90 # Prompt Catalog guardrails/refs (Prompt) — pinned
|
|
21
24
|
SKILL_BODY = 85 # <active_skill> trigger-matched body (SkillTrigger)
|
|
22
25
|
SKILL = 80 # <available_skills> level 1 (Skill)
|
|
26
|
+
KNOWLEDGE = 77 # <knowledge> top-K learned concepts (Knowledge) —
|
|
27
|
+
# below curated skills, above a single conversation's memory
|
|
23
28
|
MEMORY = 75 # <memory> read path (Memory)
|
|
24
29
|
TOOL_SEARCH = 70 # <available_tools> level 1 (ToolSearch)
|
|
25
30
|
BRIEFING = 65 # <briefing> session working state (Briefing) — D5:
|
|
@@ -27,6 +32,9 @@ module Insika
|
|
|
27
32
|
# pinned prefix), above the turn's own <request_context>.
|
|
28
33
|
HISTORY_MAX = 79 # history ceiling by recency (Session)
|
|
29
34
|
HISTORY_BASE = 60 # history base; +idx up to the ceiling (Session)
|
|
35
|
+
COMPACTION = 59 # <conversation_summary> the compacted prefix (RFC-0044)
|
|
36
|
+
# — one step below the oldest verbatim message: under
|
|
37
|
+
# budget it is the "oldest unit" and drops first.
|
|
30
38
|
REQUEST = 40 # <request_context> — turn injection, the most cuttable
|
|
31
39
|
end
|
|
32
40
|
end
|
|
@@ -5,9 +5,20 @@ module Insika
|
|
|
5
5
|
module Providers
|
|
6
6
|
# Read path for the session briefing: the per-session
|
|
7
7
|
# working-state the agent keeps and asks for. Thin adapter over the
|
|
8
|
-
# SessionStore, same pattern as Memory
|
|
9
|
-
#
|
|
10
|
-
#
|
|
8
|
+
# SessionStore, same pattern as Memory, deterministic. The MISSING list is
|
|
9
|
+
# rendered, never implied — that list is what stops the model re-asking
|
|
10
|
+
# for a field already given.
|
|
11
|
+
#
|
|
12
|
+
# TWO fragments, and the split is the whole point:
|
|
13
|
+
# · `<briefing>` (:system) — the DURABLE facts, what is already known.
|
|
14
|
+
# Head of the prompt, where reference material belongs.
|
|
15
|
+
# · `<recitation>` (:tail) — what is still missing and what the next step
|
|
16
|
+
# is, rendered AFTER the whole history, as the last thing before the
|
|
17
|
+
# current user message.
|
|
18
|
+
# Attention is strongest at the END of the context: a goal stated only in
|
|
19
|
+
# the head is the first thing a 30-call turn forgets. The recitation lives
|
|
20
|
+
# in exactly one place — it was MOVED out of the head, never duplicated, so
|
|
21
|
+
# a turn pays for it once.
|
|
11
22
|
class Briefing < ContextProvider
|
|
12
23
|
def initialize(session_store:)
|
|
13
24
|
@session_store = session_store
|
|
@@ -34,12 +45,10 @@ module Insika
|
|
|
34
45
|
declared = Array(request.profile.briefing_fields)
|
|
35
46
|
return [] if declared.empty? # defensive; enabled_for? already gates
|
|
36
47
|
|
|
37
|
-
|
|
38
|
-
|
|
48
|
+
missing = declared.reject { |name| Coercion.present?(fields[name]) }
|
|
49
|
+
next_step = briefing["next_step"]
|
|
39
50
|
|
|
40
|
-
[
|
|
41
|
-
priority: Context::Priority::BRIEFING,
|
|
42
|
-
source: id)]
|
|
51
|
+
[head_fragment(declared, fields), tail_fragment(missing, next_step)].compact
|
|
43
52
|
end
|
|
44
53
|
|
|
45
54
|
private
|
|
@@ -52,37 +61,57 @@ module Insika
|
|
|
52
61
|
@session_store.find(session.id)&.briefing || {}
|
|
53
62
|
end
|
|
54
63
|
|
|
55
|
-
#
|
|
64
|
+
# The HEAD block — durable facts only. Byte contract:
|
|
56
65
|
# <briefing>
|
|
57
66
|
# known:
|
|
58
67
|
# size: M
|
|
59
|
-
# still missing: delivery_day
|
|
60
|
-
# next step: send the payment link tomorrow at 10
|
|
61
68
|
# </briefing>
|
|
62
|
-
#
|
|
63
|
-
#
|
|
64
|
-
#
|
|
65
|
-
#
|
|
66
|
-
|
|
67
|
-
# the pack re-declares them).
|
|
68
|
-
def format_block(declared, fields, next_step)
|
|
69
|
+
# Nothing known yet -> no fragment at all (an empty `known:` header
|
|
70
|
+
# teaches the model nothing and still costs a cache invalidation).
|
|
71
|
+
# Stored keys NOT in the declaration are never rendered (they stay in the
|
|
72
|
+
# store and reappear if the pack re-declares them).
|
|
73
|
+
def head_fragment(declared, fields)
|
|
69
74
|
known = declared.filter_map do |name|
|
|
70
75
|
" #{name}: #{flatten(fields[name])}" if Coercion.present?(fields[name])
|
|
71
76
|
end
|
|
72
|
-
|
|
77
|
+
return nil if known.empty?
|
|
73
78
|
|
|
79
|
+
block = <<~BLOCK.strip
|
|
80
|
+
<briefing>
|
|
81
|
+
known:
|
|
82
|
+
#{known.join("\n")}
|
|
83
|
+
</briefing>
|
|
84
|
+
BLOCK
|
|
85
|
+
ContextFragment.build(content: block, placement: :system,
|
|
86
|
+
priority: Context::Priority::BRIEFING, source: id)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# The TAIL recitation — two lines, no more. Byte contract:
|
|
90
|
+
# <recitation>
|
|
91
|
+
# still missing: delivery_day
|
|
92
|
+
# next step: send the payment link tomorrow at 10
|
|
93
|
+
# </recitation>
|
|
94
|
+
# `still missing` renders every declared field with no stored value
|
|
95
|
+
# (including the all-missing case — that is this block's job); `next step`
|
|
96
|
+
# renders only when non-nil. Neither -> no fragment.
|
|
97
|
+
#
|
|
98
|
+
# A `user` message, like every other engine append inside a turn
|
|
99
|
+
# (LoopDetector, TurnBudget): the system prefix stays byte-stable, so the
|
|
100
|
+
# cache breakpoint at its end keeps hitting.
|
|
101
|
+
def tail_fragment(missing, next_step)
|
|
74
102
|
lines = []
|
|
75
|
-
lines << "known:" unless known.empty?
|
|
76
|
-
lines.concat(known)
|
|
77
103
|
lines << "still missing: #{missing.join(', ')}" unless missing.empty?
|
|
78
104
|
lines << "next step: #{flatten(next_step)}" if Coercion.present?(next_step)
|
|
79
105
|
return nil if lines.empty?
|
|
80
106
|
|
|
81
|
-
<<~BLOCK.strip
|
|
82
|
-
<
|
|
107
|
+
block = <<~BLOCK.strip
|
|
108
|
+
<recitation>
|
|
83
109
|
#{lines.join("\n")}
|
|
84
|
-
</
|
|
110
|
+
</recitation>
|
|
85
111
|
BLOCK
|
|
112
|
+
ContextFragment.build(content: { role: :user, content: block },
|
|
113
|
+
placement: :tail,
|
|
114
|
+
priority: Context::Priority::RECITATION, source: id)
|
|
86
115
|
end
|
|
87
116
|
|
|
88
117
|
# utf8 the value and flatten newlines/whitespace so a value can never
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Insika
|
|
4
|
+
module Context
|
|
5
|
+
module Providers
|
|
6
|
+
# Level 1 (progressive disclosure) of what the engine has LEARNED, as
|
|
7
|
+
# opposed to what a human curated (Skill) or the model wrote mid-turn
|
|
8
|
+
# (Memory). Retrieval is per-message and dynamic, so — like
|
|
9
|
+
# `SkillTrigger`, unlike the static `CatalogProvider` subclasses — this
|
|
10
|
+
# builds its `<knowledge>` block directly in `call`, never a fixed list.
|
|
11
|
+
class Knowledge < ContextProvider
|
|
12
|
+
def initialize(store:)
|
|
13
|
+
@store = store
|
|
14
|
+
# One Index PER TYPE, built once and reused for every agent/turn —
|
|
15
|
+
# never per call. This provider instance itself lives for the
|
|
16
|
+
# process's lifetime (built once at boot, see wiring), so an
|
|
17
|
+
# Index rebuilt fresh each call would throw away its own read
|
|
18
|
+
# cache (Index::Scan's dominant cost is re-parsing YAML
|
|
19
|
+
# frontmatter; measured, not assumed) on every single turn.
|
|
20
|
+
# Keyed by the config's `index` string so a future FTS5 agent
|
|
21
|
+
# gets its own instance, never Scan's.
|
|
22
|
+
@indexes = Hash.new { |h, index_name| h[index_name] = Insika::Knowledge::Index.build({ "index" => index_name }, store: @store) }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Per-agent opt-in (`knowledge.retrieve`), like Memory's `profile.memory`.
|
|
26
|
+
def enabled_for?(profile) = !!(profile.knowledge && Coercion.truthy?(profile.knowledge["retrieve"]))
|
|
27
|
+
|
|
28
|
+
# required? == false (default): a store failure degrades to a
|
|
29
|
+
# :provider_warning, never aborts the turn.
|
|
30
|
+
def call(request)
|
|
31
|
+
config = request.profile.knowledge
|
|
32
|
+
return [] unless config
|
|
33
|
+
|
|
34
|
+
top_k = positive_int(config["top_k"]) || 5
|
|
35
|
+
index = @indexes[config["index"].to_s]
|
|
36
|
+
matches = index.search(request.profile.id, tenant: request.tenant,
|
|
37
|
+
query: request.message.to_s, top_k: top_k)
|
|
38
|
+
return [] if matches.empty?
|
|
39
|
+
|
|
40
|
+
hits = matches.map { |c| [c, "top-K match"] } +
|
|
41
|
+
expand_links(matches, request, top_k).map { |c| [c, "one-hop link"] }
|
|
42
|
+
|
|
43
|
+
[ContextFragment.build(
|
|
44
|
+
content: format_block(hits), placement: :system,
|
|
45
|
+
priority: Context::Priority::KNOWLEDGE, source: id,
|
|
46
|
+
labels: hits.map { |c, reason| { "name" => c[:name], "reason" => reason } }
|
|
47
|
+
)]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def positive_int(value)
|
|
53
|
+
n = value.to_i
|
|
54
|
+
n.positive? ? n : nil
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# ONE level, deliberately — the same "cannot work without" reasoning
|
|
58
|
+
# `SkillTrigger#companions` applies to a skill's declared companions:
|
|
59
|
+
# a transitive walk would make a cycle a hang and a chain a budget
|
|
60
|
+
# blowout. Newly-discovered concepts (not already in the top-K) are
|
|
61
|
+
# capped at top_k again — "that one hop is the whole graph benefit at
|
|
62
|
+
# ~1% of the graph cost", not a second unbounded retrieval.
|
|
63
|
+
def expand_links(matches, request, top_k)
|
|
64
|
+
known = matches.map { |c| c[:name] }
|
|
65
|
+
discovered = []
|
|
66
|
+
matches.each do |concept|
|
|
67
|
+
Insika::Knowledge::Concept.links(concept[:body]).each do |name|
|
|
68
|
+
next if known.include?(name) || discovered.any? { |d| d[:name] == name }
|
|
69
|
+
|
|
70
|
+
found = fetch(request, name)
|
|
71
|
+
discovered << found if found
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
discovered.first(top_k)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def fetch(request, name)
|
|
78
|
+
raw = @store.get(request.profile.id, name, tenant: request.tenant)
|
|
79
|
+
raw && Insika::Knowledge::Concept.parse(raw)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Level 1 only — name/description/confidence/provenance, never the
|
|
83
|
+
# body (that is `load_knowledge`'s job). The instruction is the exact
|
|
84
|
+
# lesson the knowledge-adoption experiment drew: a polite "when to
|
|
85
|
+
# use" scored near zero; an explicit, ordered rule naming the tool
|
|
86
|
+
# held up. Present only when there is something to point at.
|
|
87
|
+
def format_block(hits)
|
|
88
|
+
entries = hits.map do |c, _reason|
|
|
89
|
+
%( <concept name="#{c[:name]}" confidence="#{format('%.2f', c[:confidence])}" ) +
|
|
90
|
+
%(provenance="#{c[:provenance]}">#{c[:description]}</concept>)
|
|
91
|
+
end.join("\n")
|
|
92
|
+
|
|
93
|
+
<<~BLOCK.strip
|
|
94
|
+
<knowledge>
|
|
95
|
+
#{entries}
|
|
96
|
+
</knowledge>
|
|
97
|
+
|
|
98
|
+
If the customer's question needs more than the summary above, call
|
|
99
|
+
`load_knowledge("name")` FIRST — before any other lookup for that
|
|
100
|
+
topic. This is learned from past conversations, not official
|
|
101
|
+
policy: never state a `provenance="observed"` concept to the
|
|
102
|
+
customer as a guarantee.
|
|
103
|
+
BLOCK
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
@@ -8,12 +8,16 @@ module Insika
|
|
|
8
8
|
# a WRONG agent, not a degraded one. prompt_refs: priority 90 pinned
|
|
9
9
|
# fragments, from the PromptCatalog (catalog defaults to nil).
|
|
10
10
|
#
|
|
11
|
-
# PER-AGENT identity
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
# (
|
|
15
|
-
#
|
|
16
|
-
# `
|
|
11
|
+
# PER-AGENT identity, from `profile.prompt_files`/`base_prompt` only — an
|
|
12
|
+
# agent that declares neither has no identity here, and there is no
|
|
13
|
+
# deployment-wide fallback file to borrow one from. A shared fallback
|
|
14
|
+
# (a prior design: an agent without prompt_files silently inherited a
|
|
15
|
+
# wiring-level default) is exactly how a real deployment answered as the
|
|
16
|
+
# WRONG business: a `copilot` data agent provisioned without its own
|
|
17
|
+
# identity inherited the deployment's demo persona ("Bia, Pizzaria do
|
|
18
|
+
# Zé") byte for byte, confirmed live. An agent never wears another
|
|
19
|
+
# agent's identity — `@base`/`system_files` stay legitimate (the same
|
|
20
|
+
# generic content for every agent, never one agent's specific persona).
|
|
17
21
|
class Prompt < ContextProvider
|
|
18
22
|
# Engine-owned execution discipline, appended AFTER the agent's identity.
|
|
19
23
|
# The one behavior every reference harness bakes into its base prompt
|
|
@@ -28,11 +32,12 @@ module Insika
|
|
|
28
32
|
"query, use a synonym or broader term, drop a secondary filter — before telling " \
|
|
29
33
|
"the user you found nothing. Do not narrate the retries. Then conclude.\n" \
|
|
30
34
|
"- Tool error: read the error, fix the arguments or try another path; never " \
|
|
31
|
-
"repeat the exact same call
|
|
35
|
+
"repeat the exact same call.\n" \
|
|
36
|
+
"- A URL in a tool result (e.g. a `url` field): quote it byte-for-byte. Never " \
|
|
37
|
+
"construct, guess, or rewrite the domain, host, or path."
|
|
32
38
|
|
|
33
|
-
def initialize(base: "",
|
|
39
|
+
def initialize(base: "", catalog: nil, agent_files: nil, system_files: nil)
|
|
34
40
|
@base = base
|
|
35
|
-
@files = Array(files)
|
|
36
41
|
@catalog = catalog
|
|
37
42
|
@agent_files = agent_files
|
|
38
43
|
@system_files = system_files
|
|
@@ -43,12 +48,18 @@ module Insika
|
|
|
43
48
|
def layer = :identity
|
|
44
49
|
|
|
45
50
|
def call(request)
|
|
46
|
-
fragments = []
|
|
47
51
|
identity = build_identity(request.profile)
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
52
|
+
if identity.empty?
|
|
53
|
+
raise ContextError.new(
|
|
54
|
+
"agent '#{request.profile&.id}' has no identity of its own (no base_prompt, " \
|
|
55
|
+
"no prompt_files) — refusing to run rather than answer with no identity or " \
|
|
56
|
+
"another agent's",
|
|
57
|
+
provider: id
|
|
58
|
+
)
|
|
51
59
|
end
|
|
60
|
+
|
|
61
|
+
fragments = [ContextFragment.build(content: identity, placement: :system,
|
|
62
|
+
priority: Context::Priority::IDENTITY, source: id, pinned: true)]
|
|
52
63
|
fragments.concat(ref_fragments(request.profile))
|
|
53
64
|
fragments
|
|
54
65
|
end
|
|
@@ -59,10 +70,9 @@ module Insika
|
|
|
59
70
|
# A SINGLE fragment preserves the internal base->files order (sorting
|
|
60
71
|
# acts only BETWEEN fragments) and guarantees byte-for-byte parity.
|
|
61
72
|
#
|
|
62
|
-
# profile.prompt_files
|
|
63
|
-
#
|
|
64
|
-
#
|
|
65
|
-
# order. Without prompt_files, falls back to the wiring's @files.
|
|
73
|
+
# profile.prompt_files: each source resolves via AgentFileStore (per
|
|
74
|
+
# agent) OR File.read (on-disk path) — in that order. No other agent's
|
|
75
|
+
# files are ever read for this one; there is no wiring-level fallback.
|
|
66
76
|
def build_identity(profile)
|
|
67
77
|
parts = [@base]
|
|
68
78
|
parts.concat(system_parts) # global system files, for EVERY agent
|
|
@@ -73,15 +83,11 @@ module Insika
|
|
|
73
83
|
# NO identity at all, and a chatty model answered plausibly enough to hide
|
|
74
84
|
# it. Additive to prompt_files, not exclusive: an agent may carry both.
|
|
75
85
|
parts << profile&.base_prompt.to_s
|
|
76
|
-
|
|
77
|
-
if sources.empty?
|
|
78
|
-
@files.each { |f| parts << File.read(f, encoding: "UTF-8") if File.exist?(f) }
|
|
79
|
-
else
|
|
80
|
-
sources.each { |src| parts << read_source(profile&.id, src.to_s) }
|
|
81
|
-
end
|
|
86
|
+
Array(profile&.prompt_files).each { |src| parts << read_source(profile&.id, src.to_s) }
|
|
82
87
|
identity = parts.reject { |p| p.nil? || p.strip.empty? }.join("\n\n")
|
|
83
88
|
# Discipline rides an EXISTING identity, never substitutes one: an
|
|
84
|
-
#
|
|
89
|
+
# empty identity here is fatal (`call` raises), not silently patched
|
|
90
|
+
# with the engine's own boilerplate.
|
|
85
91
|
return identity if identity.empty? || !tool_persistence?(profile)
|
|
86
92
|
|
|
87
93
|
"#{identity}\n\n#{TOOL_PERSISTENCE}"
|