@vellumai/assistant 0.8.9-staging.2 → 0.8.9-staging.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/docs/activation-funnel-telemetry.md +310 -0
  2. package/openapi.yaml +16 -115
  3. package/package.json +1 -1
  4. package/src/__tests__/activation-early-marking.test.ts +120 -0
  5. package/src/__tests__/agent-loop-output-hooks.test.ts +13 -13
  6. package/src/__tests__/anthropic-provider.test.ts +23 -8
  7. package/src/__tests__/approval-cascade.test.ts +1 -1
  8. package/src/__tests__/compaction-direct.test.ts +32 -18
  9. package/src/__tests__/compaction-events.test.ts +2 -2
  10. package/src/__tests__/compaction.benchmark.test.ts +1 -1
  11. package/src/__tests__/context-overflow-reducer.test.ts +5 -5
  12. package/src/__tests__/context-window-manager-compact-retry.test.ts +121 -15
  13. package/src/__tests__/conversation-abort-tool-results.test.ts +1 -1
  14. package/src/__tests__/conversation-confirmation-signals.test.ts +1 -1
  15. package/src/__tests__/conversation-error.test.ts +15 -1
  16. package/src/__tests__/conversation-history-web-search.test.ts +5 -0
  17. package/src/__tests__/conversation-media-retry.test.ts +1 -1
  18. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +40 -0
  19. package/src/__tests__/conversation-process-callsite.test.ts +1 -1
  20. package/src/__tests__/conversation-provider-retry-repair.test.ts +1 -1
  21. package/src/__tests__/conversation-queue.test.ts +1 -1
  22. package/src/__tests__/conversation-runtime-assembly.test.ts +71 -0
  23. package/src/__tests__/conversation-slash-queue.test.ts +1 -1
  24. package/src/__tests__/conversation-slash-unknown.test.ts +1 -1
  25. package/src/__tests__/conversation-speed-override.test.ts +1 -1
  26. package/src/__tests__/conversation-surfaces-activation-emit.test.ts +395 -0
  27. package/src/__tests__/conversation-surfaces-app-control.test.ts +44 -0
  28. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +73 -4
  29. package/src/__tests__/conversation-undo.test.ts +2 -2
  30. package/src/__tests__/conversation-workspace-cache-state.test.ts +1 -1
  31. package/src/__tests__/conversation-workspace-injection.test.ts +1 -1
  32. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +1 -1
  33. package/src/__tests__/credential-security-invariants.test.ts +1 -0
  34. package/src/__tests__/cu-unified-flow.test.ts +36 -0
  35. package/src/__tests__/history-repair-hook.test.ts +2 -0
  36. package/src/__tests__/llm-resolver.test.ts +73 -0
  37. package/src/__tests__/memory-retrieval-hook.test.ts +1 -2
  38. package/src/__tests__/persist-unsendable-image-downscale.test.ts +145 -0
  39. package/src/__tests__/persist-unsendable-image.test.ts +97 -1
  40. package/src/__tests__/plugin-external-api.test.ts +68 -0
  41. package/src/__tests__/post-turn-tool-result-truncation.test.ts +69 -0
  42. package/src/__tests__/published-app-updater.test.ts +138 -0
  43. package/src/__tests__/skill-feature-flags-integration.test.ts +5 -7
  44. package/src/__tests__/title-generate-hook.test.ts +2 -0
  45. package/src/__tests__/web-fetch.test.ts +45 -0
  46. package/src/acp/__tests__/helpers/acp-config-stub.ts +0 -2
  47. package/src/acp/resolve-agent.test.ts +0 -56
  48. package/src/acp/resolve-agent.ts +10 -38
  49. package/src/agent/loop.ts +13 -27
  50. package/src/api/responses/memory-v3-selection-log.ts +19 -10
  51. package/src/cli/commands/__tests__/memory-v3.test.ts +191 -210
  52. package/src/cli/commands/memory-v3.ts +57 -199
  53. package/src/cli/lib/__tests__/install-from-github.test.ts +232 -29
  54. package/src/cli/lib/__tests__/plugin-details.test.ts +28 -19
  55. package/src/cli/lib/__tests__/plugin-marketplace.test.ts +57 -7
  56. package/src/cli/lib/__tests__/search-plugins.test.ts +17 -10
  57. package/src/cli/lib/install-from-github.ts +258 -41
  58. package/src/cli/lib/plugin-details.ts +20 -13
  59. package/src/cli/lib/plugin-marketplace.ts +23 -5
  60. package/src/cli/lib/search-plugins.ts +14 -8
  61. package/src/config/acp-defaults.ts +3 -3
  62. package/src/config/acp-schema.ts +1 -7
  63. package/src/config/bundled-skills/acp/SKILL.md +4 -17
  64. package/src/config/bundled-skills/acp/TOOLS.json +2 -2
  65. package/src/config/call-site-defaults.ts +0 -1
  66. package/src/config/feature-flag-registry.json +3 -18
  67. package/src/config/llm-resolver.ts +39 -7
  68. package/src/config/schemas/__tests__/memory-v3.test.ts +25 -9
  69. package/src/config/schemas/call-site-catalog.ts +0 -7
  70. package/src/config/schemas/llm.ts +17 -1
  71. package/src/config/schemas/memory-v3.ts +58 -8
  72. package/src/config/seed-inference-profiles.ts +18 -0
  73. package/src/context/post-turn-tool-result-truncation.ts +39 -1
  74. package/src/daemon/conversation-agent-loop-handlers.ts +8 -1
  75. package/src/daemon/conversation-agent-loop.ts +87 -95
  76. package/src/daemon/conversation-error.ts +31 -4
  77. package/src/daemon/conversation-history.ts +1 -1
  78. package/src/daemon/conversation-media-retry.ts +19 -6
  79. package/src/daemon/conversation-messaging.ts +17 -0
  80. package/src/daemon/conversation-process.ts +14 -5
  81. package/src/daemon/conversation-queue-manager.ts +8 -0
  82. package/src/daemon/conversation-runtime-assembly.ts +37 -1
  83. package/src/daemon/conversation-surfaces.ts +141 -3
  84. package/src/daemon/conversation.ts +48 -13
  85. package/src/daemon/external-plugins-bootstrap.ts +8 -3
  86. package/src/daemon/persist-unsendable-image.ts +62 -25
  87. package/src/daemon/process-message.ts +1 -1
  88. package/src/daemon/tool-side-effects.ts +15 -0
  89. package/src/memory/__tests__/activation-session-store.test.ts +41 -0
  90. package/src/memory/__tests__/onboarding-events-store.test.ts +80 -0
  91. package/src/memory/activation-session-store.ts +43 -0
  92. package/src/memory/db-init.ts +4 -0
  93. package/src/memory/migrations/273-onboarding-events-funnel-columns.ts +46 -0
  94. package/src/memory/migrations/274-create-activation-sessions.ts +15 -0
  95. package/src/memory/migrations/index.ts +2 -0
  96. package/src/memory/onboarding-events-store.ts +66 -18
  97. package/src/memory/schema/infrastructure.ts +13 -0
  98. package/src/memory/v2/__tests__/consolidation-job.test.ts +4 -97
  99. package/src/memory/v2/__tests__/page-store.test.ts +22 -0
  100. package/src/memory/v2/consolidation-job.ts +2 -72
  101. package/src/memory/v2/types.ts +5 -0
  102. package/src/messaging/providers/telegram-bot/api.ts +14 -5
  103. package/src/notifications/adapters/telegram.ts +7 -1
  104. package/src/plugin-api/constants.ts +2 -2
  105. package/src/plugin-api/index.ts +2 -2
  106. package/src/plugin-api/types.ts +19 -5
  107. package/src/plugins/defaults/compaction/compact.ts +24 -15
  108. package/src/plugins/defaults/compaction/context-overflow-reducer.ts +4 -4
  109. package/src/plugins/defaults/compaction/manager-store.ts +1 -1
  110. package/src/{context → plugins/defaults/compaction}/window-manager.ts +68 -12
  111. package/src/plugins/defaults/memory-retrieval/hooks/user-prompt-submit-temp.ts +12 -18
  112. package/src/plugins/defaults/memory-v3-shadow/__tests__/capabilities.test.ts +19 -66
  113. package/src/plugins/defaults/memory-v3-shadow/__tests__/dense.test.ts +181 -0
  114. package/src/plugins/defaults/memory-v3-shadow/__tests__/edge.test.ts +247 -0
  115. package/src/plugins/defaults/memory-v3-shadow/__tests__/live-integration.test.ts +139 -129
  116. package/src/plugins/defaults/memory-v3-shadow/__tests__/maintain-job.test.ts +332 -164
  117. package/src/plugins/defaults/memory-v3-shadow/__tests__/orchestrate.test.ts +516 -293
  118. package/src/plugins/defaults/memory-v3-shadow/__tests__/pool-select.test.ts +306 -0
  119. package/src/plugins/defaults/memory-v3-shadow/__tests__/render-injection.test.ts +51 -21
  120. package/src/plugins/defaults/memory-v3-shadow/__tests__/section-dense-store.test.ts +402 -0
  121. package/src/plugins/defaults/memory-v3-shadow/__tests__/section-needle.test.ts +135 -0
  122. package/src/plugins/defaults/memory-v3-shadow/__tests__/sections.test.ts +125 -0
  123. package/src/plugins/defaults/memory-v3-shadow/__tests__/selection-log-store.test.ts +66 -11
  124. package/src/plugins/defaults/memory-v3-shadow/__tests__/shadow-integration.test.ts +446 -0
  125. package/src/plugins/defaults/memory-v3-shadow/__tests__/shadow-plugin.test.ts +271 -110
  126. package/src/plugins/defaults/memory-v3-shadow/__tests__/types.test.ts +0 -22
  127. package/src/plugins/defaults/memory-v3-shadow/capabilities.ts +26 -62
  128. package/src/plugins/defaults/memory-v3-shadow/dense.ts +97 -0
  129. package/src/plugins/defaults/memory-v3-shadow/edge.ts +252 -0
  130. package/src/plugins/defaults/memory-v3-shadow/injector.ts +3 -2
  131. package/src/plugins/defaults/memory-v3-shadow/maintain-job.ts +415 -181
  132. package/src/plugins/defaults/memory-v3-shadow/orchestrate.ts +196 -56
  133. package/src/plugins/defaults/memory-v3-shadow/page-content.ts +39 -2
  134. package/src/plugins/defaults/memory-v3-shadow/pool-select.ts +204 -0
  135. package/src/plugins/defaults/memory-v3-shadow/render-injection.ts +15 -7
  136. package/src/plugins/defaults/memory-v3-shadow/section-dense-store.ts +236 -0
  137. package/src/plugins/defaults/memory-v3-shadow/section-needle.ts +200 -0
  138. package/src/plugins/defaults/memory-v3-shadow/sections.ts +115 -0
  139. package/src/plugins/defaults/memory-v3-shadow/selection-log-store.ts +75 -3
  140. package/src/plugins/defaults/memory-v3-shadow/shadow-plugin.ts +111 -78
  141. package/src/plugins/defaults/memory-v3-shadow/types.ts +52 -19
  142. package/src/plugins/defaults/memory-v3-shadow/working-set.ts +4 -1
  143. package/src/plugins/external-api.ts +114 -0
  144. package/src/prompts/system-prompt.ts +61 -10
  145. package/src/prompts/templates/BOOTSTRAP-ACTIVATION-RAIL.md +37 -2
  146. package/src/providers/anthropic/client.ts +9 -10
  147. package/src/providers/inference/kimi-cjk-token-ids.ts +493 -0
  148. package/src/providers/inference/logit-bias.ts +55 -0
  149. package/src/providers/openai/__tests__/vision-not-supported.test.ts +75 -0
  150. package/src/providers/openai/chat-completions-provider.ts +44 -1
  151. package/src/providers/retry.ts +22 -0
  152. package/src/providers/types.ts +6 -0
  153. package/src/runtime/routes/__tests__/stt-routes.test.ts +112 -0
  154. package/src/runtime/routes/acp-routes.test.ts +3 -20
  155. package/src/runtime/routes/app-management-routes.ts +3 -0
  156. package/src/runtime/routes/conversation-routes.ts +2 -0
  157. package/src/runtime/routes/memory-v3-routes.ts +64 -314
  158. package/src/runtime/routes/playground/__tests__/force-compact.test.ts +1 -1
  159. package/src/runtime/routes/stt-routes.ts +45 -12
  160. package/src/runtime/routes/workspace-routes.ts +50 -15
  161. package/src/services/published-app-updater.ts +30 -8
  162. package/src/telemetry/__tests__/activation-funnel.test.ts +95 -0
  163. package/src/telemetry/activation-funnel.ts +167 -0
  164. package/src/telemetry/types.ts +13 -0
  165. package/src/telemetry/usage-telemetry-reporter.test.ts +154 -0
  166. package/src/telemetry/usage-telemetry-reporter.ts +26 -1
  167. package/src/tools/acp/list-agents.test.ts +2 -18
  168. package/src/tools/acp/list-agents.ts +3 -15
  169. package/src/tools/acp/spawn.test.ts +0 -10
  170. package/src/tools/browser/browser-execution.ts +12 -2
  171. package/src/tools/network/web-fetch.ts +65 -24
  172. package/src/tools/skills/load.ts +1 -1
  173. package/src/tools/ui-surface/definitions.ts +7 -0
  174. package/src/acp/feature-gate.test.ts +0 -48
  175. package/src/acp/feature-gate.ts +0 -34
  176. package/src/plugins/defaults/memory-v3-shadow/__tests__/assign.test.ts +0 -242
  177. package/src/plugins/defaults/memory-v3-shadow/__tests__/core.test.ts +0 -39
  178. package/src/plugins/defaults/memory-v3-shadow/__tests__/health.test.ts +0 -219
  179. package/src/plugins/defaults/memory-v3-shadow/__tests__/needle.test.ts +0 -107
  180. package/src/plugins/defaults/memory-v3-shadow/__tests__/provider-blocks.test.ts +0 -13
  181. package/src/plugins/defaults/memory-v3-shadow/__tests__/reconcile.test.ts +0 -274
  182. package/src/plugins/defaults/memory-v3-shadow/__tests__/router.test.ts +0 -337
  183. package/src/plugins/defaults/memory-v3-shadow/__tests__/selector.test.ts +0 -470
  184. package/src/plugins/defaults/memory-v3-shadow/__tests__/snapshot.test.ts +0 -168
  185. package/src/plugins/defaults/memory-v3-shadow/__tests__/tree.test.ts +0 -192
  186. package/src/plugins/defaults/memory-v3-shadow/assign.ts +0 -272
  187. package/src/plugins/defaults/memory-v3-shadow/core.ts +0 -26
  188. package/src/plugins/defaults/memory-v3-shadow/health.ts +0 -0
  189. package/src/plugins/defaults/memory-v3-shadow/needle.ts +0 -115
  190. package/src/plugins/defaults/memory-v3-shadow/provider-blocks.ts +0 -26
  191. package/src/plugins/defaults/memory-v3-shadow/reconcile.ts +0 -527
  192. package/src/plugins/defaults/memory-v3-shadow/router.ts +0 -190
  193. package/src/plugins/defaults/memory-v3-shadow/selector.ts +0 -226
  194. package/src/plugins/defaults/memory-v3-shadow/snapshot.ts +0 -209
  195. package/src/plugins/defaults/memory-v3-shadow/tree.ts +0 -174
  196. package/src/util/map-limit.ts +0 -27
@@ -15,9 +15,10 @@
15
15
  * - both OFF: orchestration is skipped entirely.
16
16
  *
17
17
  * On each turn (either flag on):
18
- * 1. Lazy-init the v3 lanes ONCE across the whole process (leaf tree, core
19
- * set, BM25 needle, carry-forward working set), memoizing the init
20
- * promise so concurrent first turns share a single build.
18
+ * 1. Lazy-init the v3 lanes ONCE across the whole process (section index,
19
+ * section-grain BM25 needle, dense lane config, link-graph edge graph,
20
+ * carry-forward working set), memoizing the init promise so concurrent
21
+ * first turns share a single build.
21
22
  * 2. Build a {@link MemoryRoutingTurn} from the conversation's recent messages.
22
23
  * 3. Run {@link orchestrate} and record its selection set to
23
24
  * `memory_v3_selections` with a best-effort lane attribution.
@@ -37,23 +38,25 @@ import { getMessages } from "../../../memory/conversation-crud.js";
37
38
  import { getDb, getSqliteFrom } from "../../../memory/db-connection.js";
38
39
  import { stringifyMessageContent } from "../../../memory/message-content.js";
39
40
  import { getPageIndex } from "../../../memory/v2/page-index.js";
41
+ import { readPage, renderPageContent } from "../../../memory/v2/page-store.js";
40
42
  import { getLogger } from "../../../util/logger.js";
41
43
  import {
42
44
  getWorkspaceDir,
43
45
  getWorkspacePromptPath,
44
46
  } from "../../../util/platform.js";
45
47
  import { stripCommentLines } from "../../../util/strip-comment-lines.js";
46
- import { injectCapabilitiesLeaf, isCapabilitySlug } from "./capabilities.js";
47
- import { loadCore } from "./core.js";
48
- import type { NeedleIndex } from "./needle.js";
49
- import { buildNeedleIndex } from "./needle.js";
48
+ import { capabilityOrDiskBody } from "./capabilities.js";
49
+ import type { EdgeGraph } from "./edge.js";
50
+ import { buildEdgeGraph } from "./edge.js";
50
51
  import type { OrchestrateResult } from "./orchestrate.js";
51
52
  import { orchestrate } from "./orchestrate.js";
52
- import { coreSlugs, loadLeafTree, resolveDataDir } from "./tree.js";
53
+ import { ensureSectionCollection } from "./section-dense-store.js";
54
+ import type { SectionNeedle } from "./section-needle.js";
55
+ import { buildSectionNeedle } from "./section-needle.js";
56
+ import { buildSectionIndex } from "./sections.js";
53
57
  import {
54
- type LeafPath,
55
- type LeafTree,
56
58
  type MemoryRoutingTurn,
59
+ type SectionIndex,
57
60
  type SelectionSource,
58
61
  type Slug,
59
62
  } from "./types.js";
@@ -74,26 +77,29 @@ const RECENT_CONTEXT_MESSAGES = 6;
74
77
  * carry-forward lane.
75
78
  */
76
79
  export interface ShadowLanes {
77
- tree: LeafTree;
78
- core: Set<LeafPath>;
79
- needle: NeedleIndex;
80
+ sectionIndex: SectionIndex;
81
+ needle: SectionNeedle;
82
+ /** Config the dense lane needs to embed the query + search the section
83
+ * collection. */
84
+ denseConfig: AssistantConfig;
85
+ edgeGraph: EdgeGraph;
80
86
  workingSet: WorkingSet;
81
87
  }
82
88
 
83
89
  /**
84
90
  * Memoized init promise. Caching the PROMISE (not the resolved value) means
85
91
  * concurrent first turns all await the same build instead of racing several
86
- * `loadLeafTree`/`buildNeedleIndex` passes.
92
+ * section-index / needle / edge-graph passes.
87
93
  */
88
94
  let lanesPromise: Promise<ShadowLanes> | null = null;
89
95
 
90
96
  /**
91
97
  * Drop the memoized lanes so the NEXT `getLanes` rebuilds them from scratch
92
- * (fresh topic-tree load + fresh needle index). The rebuild is lazy — this only
93
- * clears the cache, so the cost is paid by the next caller, and concurrent
94
- * first-callers after the invalidation still share a single build via the
95
- * re-memoized promise. Call this whenever the underlying tree or needle sources
96
- * change on disk.
98
+ * (fresh section index + fresh needle + fresh edge graph). The rebuild is lazy
99
+ * — this only clears the cache, so the cost is paid by the next caller, and
100
+ * concurrent first-callers after the invalidation still share a single build via
101
+ * the re-memoized promise. Call this whenever the underlying pages change on
102
+ * disk.
97
103
  */
98
104
  export function invalidateLanes(): void {
99
105
  lanesPromise = null;
@@ -104,48 +110,78 @@ export function resetShadowLanesForTests(): void {
104
110
  invalidateLanes();
105
111
  }
106
112
 
107
- /**
108
- * Pull a page's summary from the existing v2 page index. Missing summaries
109
- * (and any read failure) degrade to "" — the needle and L2 selector treat an
110
- * empty summary as "no signal" rather than throwing.
111
- */
112
- async function pageSummary(slug: Slug): Promise<string> {
113
- try {
114
- const index = await getPageIndex(getWorkspaceDir());
115
- return index.bySlug.get(slug)?.summary ?? "";
116
- } catch {
117
- return "";
118
- }
119
- }
120
-
121
113
  async function initLanes(config: AssistantConfig): Promise<ShadowLanes> {
122
- const dataDir = resolveDataDir();
123
114
  const pageIndex = await getPageIndex(getWorkspaceDir());
124
- const pageLeaves = new Map<Slug, LeafPath[]>();
125
- for (const entry of pageIndex.entries) {
126
- pageLeaves.set(entry.slug, entry.leaves);
115
+ const slugs = pageIndex.entries.map((entry) => entry.slug);
116
+
117
+ // Read each page ONCE and feed BOTH forms downstream: the frontmatter-stripped
118
+ // body to the section index (lexical/dense matching), and the raw page
119
+ // (frontmatter + body) to the edge graph (so the `links:` frontmatter is
120
+ // available). A per-slug cache holds the parsed page so the second consumer
121
+ // reuses the first read.
122
+ const pageCache = new Map<Slug, { body: string; raw: string } | null>();
123
+ async function loadPage(
124
+ slug: Slug,
125
+ ): Promise<{ body: string; raw: string } | null> {
126
+ if (pageCache.has(slug)) return pageCache.get(slug)!;
127
+ let loaded: { body: string; raw: string } | null = null;
128
+ try {
129
+ const page = await readPage(getWorkspaceDir(), slug);
130
+ if (page) loaded = { body: page.body, raw: renderPageContent(page) };
131
+ } catch {
132
+ loaded = null;
133
+ }
134
+ pageCache.set(slug, loaded);
135
+ return loaded;
127
136
  }
128
- const tree = await loadLeafTree(dataDir, pageLeaves);
129
- const core = await loadCore(dataDir);
137
+ // Synthetic capability slugs (skills / CLI commands) have no on-disk page, so
138
+ // they contribute their rendered capability content to the section index —
139
+ // exactly the content `page-content.ts` injects for them. This puts them in
140
+ // the section index, so the needle lane (and, once a backfill embeds them, the
141
+ // dense lane) ranks them by relevance like any other page, instead of being
142
+ // blindly added to the select pool every turn. Real pages read their body
143
+ // through the cached `loadPage`.
144
+ const pageBody = async (slug: Slug): Promise<string> =>
145
+ capabilityOrDiskBody(slug, async (s) => (await loadPage(s))?.body ?? "");
146
+ const pageRaw = async (slug: Slug): Promise<string> => {
147
+ const loaded = await loadPage(slug);
148
+ if (!loaded) throw new Error(`page not found: ${slug}`);
149
+ return loaded.raw;
150
+ };
130
151
 
131
- // Always-on synthetic capabilities leaf: skill + CLI-command rows the page
132
- // index already carries (with summaries). Injecting them as leaf members puts
133
- // them in the needle corpus (`tree.byPage`) and, via `core`, makes L1 always
134
- // open the leaf so L2 selects the relevant subset each turn — matching v2's
135
- // "always in the pool, router-selected" capability surfacing. Done before
136
- // `buildNeedleIndex` so the synthetic members are indexed.
137
- const capabilitySlugs = pageIndex.entries
138
- .map((entry) => entry.slug)
139
- .filter(isCapabilitySlug);
140
- injectCapabilitiesLeaf(tree, core, capabilitySlugs);
152
+ const sectionIndex = await buildSectionIndex(slugs, pageBody);
153
+ const needle = buildSectionNeedle(sectionIndex);
154
+ const edgeGraph = await buildEdgeGraph(pageIndex.entries, pageRaw, {
155
+ hubDegree: config.memory.v3.edge.hubDegree,
156
+ });
157
+ // Ensuring the dense collection is best-effort: the needle + edge lanes and
158
+ // carry-forward are in-memory and independent of Qdrant, so a Qdrant outage
159
+ // must NOT reject lane init (which would return `null` from observeTurn and
160
+ // disable ALL of v3, plus poison the memoized lanes until invalidation). On
161
+ // failure we log and continue with the dense lane degraded — denseLane already
162
+ // returns no hits on a Qdrant error, and maintain/backfill re-ensure the
163
+ // collection once Qdrant recovers.
164
+ try {
165
+ await ensureSectionCollection(config);
166
+ } catch (err) {
167
+ log.warn(
168
+ { err: err instanceof Error ? err.message : String(err) },
169
+ "memory-v3: section collection ensure failed; continuing with the dense lane degraded",
170
+ );
171
+ }
141
172
 
142
- const needle = await buildNeedleIndex(tree, pageSummary);
143
173
  const workingSet = new WorkingSet(
144
174
  config.memory.v3.workingSet.maxPages,
145
175
  config.memory.v3.workingSet.evictWindow,
146
176
  );
147
177
 
148
- return { tree, core, needle, workingSet };
178
+ return {
179
+ sectionIndex,
180
+ needle,
181
+ denseConfig: config,
182
+ edgeGraph,
183
+ workingSet,
184
+ };
149
185
  }
150
186
 
151
187
  /** Lazy, memoized accessor for the shadow lanes. */
@@ -180,11 +216,11 @@ function readNowContext(): string | null {
180
216
  }
181
217
 
182
218
  /**
183
- * Compose the situational signal threaded into L1 routing and L2 selection: the
184
- * current date plus the live NOW.md scratchpad. The date alone is a weak signal,
185
- * but together with the scratchpad it lets retrieval surface a leaf the message
186
- * never names (e.g. an anniversary that falls today). Always returns at least
187
- * the date line — this mirrors the `c_now`/NOW.md signal the v2 retriever uses.
219
+ * Compose the situational signal threaded into pool selection: the current date
220
+ * plus the live NOW.md scratchpad. The date alone is a weak signal, but together
221
+ * with the scratchpad it lets retrieval surface a page the message never names
222
+ * (e.g. an anniversary that falls today). Always returns at least the date line
223
+ * — this mirrors the `c_now`/NOW.md signal the v2 retriever uses.
188
224
  */
189
225
  function buildSituationalContext(): string {
190
226
  const now = readNowContext();
@@ -237,32 +273,24 @@ interface SelectionRow {
237
273
  }
238
274
 
239
275
  /**
240
- * Map an orchestrate result onto telemetry rows with best-effort lane
276
+ * Map an orchestrate result onto telemetry rows with precise per-lane source
241
277
  * attribution.
242
278
  *
243
- * - A current selection whose page belongs to a core leaf → `"core+l2"`,
244
- * otherwise `"l1+l2"`.
279
+ * - A current selection is tagged with the candgen lane that FIRST surfaced its
280
+ * page this turn — `result.laneBySlug` (`"needle"` / `"dense"` / `"edge"`),
281
+ * recorded at pool-build time. (`"needle"` is the fallback if a selected slug
282
+ * is somehow absent from the lane map, which should not happen since every
283
+ * pooled candidate is recorded there.)
245
284
  * - A slug in `finalInjection` but NOT re-selected this turn → `"carry-forward"`.
246
- *
247
- * Precise needle attribution is a documented follow-up: the needle lane only
248
- * widens the open set, so a needle-surfaced page that survives L2 selection is
249
- * indistinguishable here from an L1-routed one. This coarse mapping is
250
- * acceptable for v0 shadow telemetry.
251
285
  */
252
- function attributeSelections(
253
- tree: LeafTree,
254
- core: Set<LeafPath>,
255
- result: OrchestrateResult,
256
- ): SelectionRow[] {
257
- const coreOwnedSlugs = coreSlugs(tree, core);
258
-
286
+ export function attributeSelections(result: OrchestrateResult): SelectionRow[] {
259
287
  const rows: SelectionRow[] = [];
260
288
  const seen = new Set<Slug>();
261
289
  for (const sel of result.currentSelections) {
262
290
  seen.add(sel.slug);
263
291
  rows.push({
264
292
  slug: sel.slug,
265
- source: coreOwnedSlugs.has(sel.slug) ? "core+l2" : "l1+l2",
293
+ source: result.laneBySlug.get(sel.slug) ?? "needle",
266
294
  pinned: sel.pinned ? 1 : 0,
267
295
  });
268
296
  }
@@ -275,7 +303,7 @@ function attributeSelections(
275
303
  }
276
304
 
277
305
  /** Write the attributed selection rows to `memory_v3_selections`. */
278
- function writeSelections(
306
+ export function writeSelections(
279
307
  conversationId: string,
280
308
  turn: number,
281
309
  rows: SelectionRow[],
@@ -311,16 +339,21 @@ export async function observeTurn(
311
339
 
312
340
  const cfg = getConfig();
313
341
  const lanes = await getLanes(cfg);
342
+ const v3 = cfg.memory.v3;
314
343
  const result = await orchestrate(turn, {
315
- tree: lanes.tree,
316
- core: lanes.core,
344
+ sectionIndex: lanes.sectionIndex,
317
345
  needle: lanes.needle,
346
+ denseConfig: lanes.denseConfig,
347
+ edgeGraph: lanes.edgeGraph,
318
348
  workingSet: lanes.workingSet,
319
- pageSummary,
320
- l2Concurrency: cfg.memory.v3.l2Concurrency,
349
+ needleK: v3.needleK,
350
+ denseK: v3.denseK,
351
+ edgeSeeds: v3.edge.seedCount,
352
+ edgePerSeed: v3.edge.perSeed,
353
+ edgeCap: v3.edge.cap,
321
354
  });
322
355
 
323
- const rows = attributeSelections(lanes.tree, lanes.core, result);
356
+ const rows = attributeSelections(result);
324
357
  writeSelections(conversationId, turnIndex, rows);
325
358
  return result;
326
359
  } catch (err) {
@@ -1,4 +1,3 @@
1
- export type LeafPath = string; // dot- or slash-pathed taxonomy node
2
1
  export type Slug = string;
3
2
 
4
3
  /**
@@ -11,27 +10,34 @@ export type Slug = string;
11
10
  */
12
11
  export const MEMORY_V3_BLOCK_ID = "memory-v3" as const;
13
12
 
14
- export interface LeafFrontmatter {
15
- path: LeafPath;
16
- in_core: boolean;
17
- /**
18
- * Optional stable identifier for the leaf, independent of its taxonomy path.
19
- * Older leaves predate this field and omit it, so it is optional.
20
- */
21
- id?: string;
13
+ /**
14
+ * A single section of a page: the lead (text before the first `## heading`,
15
+ * ordinal 0) or a heading-delimited block. Over-long sections are split into
16
+ * multiple ordered `Section`s, each with its own consecutive `ordinal`, so each
17
+ * fits a typical embedding window. `text` is prefixed with a
18
+ * `${lastSlugSegment} — ${title}` head line for lexical/dense matching.
19
+ */
20
+ export interface Section {
21
+ article: Slug;
22
+ title: string;
23
+ text: string;
24
+ ordinal: number;
22
25
  }
23
26
 
24
- export interface LeafNode {
25
- path: LeafPath;
26
- frontmatter: LeafFrontmatter;
27
- description: string;
28
- members: Slug[];
29
- domain: string;
27
+ /**
28
+ * A flat, deterministic index of every page's sections plus an article→section
29
+ * lookup. `byArticle` maps each article slug to the indices (into `sections`)
30
+ * of that article's sections, in order.
31
+ */
32
+ export interface SectionIndex {
33
+ sections: Section[];
34
+ byArticle: Map<Slug, number[]>;
30
35
  }
31
36
 
32
- export interface LeafTree {
33
- leaves: Map<LeafPath, LeafNode>;
34
- byPage: Map<Slug, LeafPath[]>;
37
+ /** A page selected from the candidate pool, with whether the turn centers on it. */
38
+ export interface SelectedPage {
39
+ slug: Slug;
40
+ pinned: boolean;
35
41
  }
36
42
 
37
43
  export interface WorkingSetEntry {
@@ -56,4 +62,31 @@ export interface MemoryRoutingTurn {
56
62
  situationalContext?: string;
57
63
  }
58
64
 
59
- export type SelectionSource = "l1+l2" | "core+l2" | "needle" | "carry-forward";
65
+ /**
66
+ * Canonical ordered list of the lane sources recorded per selection. The
67
+ * {@link SelectionSource} type is derived from this so a new lane is added in
68
+ * exactly one place and the runtime list (used for telemetry roll-ups and
69
+ * source validation) can never drift from the type.
70
+ *
71
+ * The `memory_v3_selections.source` column is free-text, so tightening this set
72
+ * needs no migration: any historical rows with older labels still read back
73
+ * fine via the permissive `z.string()` row schema.
74
+ */
75
+ export const SELECTION_SOURCES = [
76
+ "needle",
77
+ "dense",
78
+ "edge",
79
+ "carry-forward",
80
+ ] as const;
81
+
82
+ export type SelectionSource = (typeof SELECTION_SOURCES)[number];
83
+
84
+ /**
85
+ * The candidate-generation lanes — the strict subset of {@link SelectionSource}
86
+ * a pooled candidate can be tagged with at pool-build time. (`carry-forward` is
87
+ * the one source assigned later, by the orchestrator's working-set union, not by
88
+ * a candgen lane.) Defined as `Exclude<SelectionSource, "carry-forward">` so it
89
+ * can never drift from {@link SELECTION_SOURCES}: adding a lane there widens this
90
+ * automatically.
91
+ */
92
+ export type CandidateLane = Exclude<SelectionSource, "carry-forward">;
@@ -42,13 +42,16 @@ export class WorkingSet {
42
42
  /**
43
43
  * Prune the working set, in order:
44
44
  * 1. Core slugs never belong here — drop any entry whose slug is core.
45
+ * The section-lane pipeline has no core set, so `coreSlugs` defaults to
46
+ * empty (no-op); the tree pipeline passed its core slugs here. Stale and
47
+ * cap eviction semantics are identical either way.
45
48
  * 2. Pinned entries never evict.
46
49
  * 3. Window eviction: drop a non-pinned entry unseen for more than
47
50
  * `evictWindow` turns.
48
51
  * 4. Cap eviction: while over `maxPages`, evict the lowest-scoring
49
52
  * non-pinned entry first (ties broken deterministically).
50
53
  */
51
- evict(currentTurn: number, coreSlugs: Set<Slug>): void {
54
+ evict(currentTurn: number, coreSlugs: Set<Slug> = new Set()): void {
52
55
  // 1. Core slugs are owned by core, not the working set.
53
56
  for (const slug of this.entries.keys()) {
54
57
  if (coreSlugs.has(slug)) {
@@ -0,0 +1,114 @@
1
+ /**
2
+ * External plugin runtime — `globalThis.__vellumPluginRuntime` bridge.
3
+ *
4
+ * ⚠️ DO NOT DELETE THIS AS "UNUSED". There is intentionally NO in-repo consumer:
5
+ * the consumers are out-of-repo, user-authored workspace plugins under
6
+ * `<workspaceDir>/plugins/*`, which a repo-wide reference search cannot see. On
7
+ * a `bun --compile` daemon (the shipped macOS app) this bridge is the ONLY way
8
+ * such a plugin can reach the daemon's real `assistantEventHub` /
9
+ * `getSecureKeyAsync` singletons — an absolute-path `import` re-loads a disjoint
10
+ * copy (see the module-identity note below). Removing it silently breaks every
11
+ * event-subscribing workspace plugin on the next app rebuild. This was already
12
+ * removed once on the false "unused" assumption (PR #33825) and restored here.
13
+ *
14
+ * Workspace-local plugins (`<workspaceDir>/plugins/*`) get dynamic-imported by
15
+ * {@link loadUserPlugins}. They need to call `registerPlugin()` from their
16
+ * module body and they typically also want to read secrets, subscribe to
17
+ * runtime events, etc.
18
+ *
19
+ * Importing those symbols by absolute path
20
+ * (`/abs/path/to/assistant/src/plugins/registry.js`) works when the daemon is
21
+ * running from source — both the daemon and the dynamic-imported plugin
22
+ * resolve the same on-disk file and share module identity. It DOES NOT work
23
+ * when the daemon is a `bun --compile` binary: the daemon's modules are baked
24
+ * into the executable, and any absolute-path import re-loads a fresh disk
25
+ * copy. `registerPlugin()` then writes into a disjoint registry instance and
26
+ * the daemon never sees the plugin.
27
+ *
28
+ * The fix is to expose a single, stable handle on `globalThis` that plugins
29
+ * read at module-load time. The daemon's bundled modules attach themselves
30
+ * here once at startup; plugins consume the same instance regardless of how
31
+ * the daemon was built.
32
+ *
33
+ * Plugins use the bridge like this:
34
+ *
35
+ * const runtime = (globalThis as { __vellumPluginRuntime?: ... })
36
+ * .__vellumPluginRuntime;
37
+ * if (!runtime || runtime.version !== 1) throw new Error("...");
38
+ * const { registerPlugin, assistantEventHub, getSecureKeyAsync } = runtime;
39
+ *
40
+ * Type-only imports (`import type { Plugin } from "..."`) remain free to use
41
+ * absolute paths or workspace-local copies — the TypeScript compiler erases
42
+ * them and they have no module-identity effect at runtime.
43
+ *
44
+ * See `experimental/plugins/README.md` for the full authoring contract and
45
+ * `assistant/examples/plugins/echo/` for a worked example.
46
+ */
47
+
48
+ import { assistantEventHub } from "../runtime/assistant-event-hub.js";
49
+ import { getSecureKeyAsync } from "../security/secure-keys.js";
50
+ import { registerPlugin } from "./registry.js";
51
+
52
+ /**
53
+ * The handle plugins read from `globalThis.__vellumPluginRuntime`.
54
+ *
55
+ * `version` is the contract version. Plugins should assert against the
56
+ * version they were authored against and refuse to register if they see a
57
+ * different one — bumping it is a breaking change to the plugin surface.
58
+ *
59
+ * The field set is intentionally small: the most-commonly-needed symbols
60
+ * across both static (module-load-time) and dynamic (init-time) plugin
61
+ * lifecycle. Plugins that need richer runtime state can still receive it
62
+ * through {@link PluginInitContext} during `init()`.
63
+ */
64
+ export interface VellumPluginRuntime {
65
+ readonly version: 1;
66
+ readonly registerPlugin: typeof registerPlugin;
67
+ readonly assistantEventHub: typeof assistantEventHub;
68
+ readonly getSecureKeyAsync: typeof getSecureKeyAsync;
69
+ }
70
+
71
+ /** Stable globalThis key. Don't rename — plugins reference it by string. */
72
+ const RUNTIME_GLOBAL_KEY = "__vellumPluginRuntime" as const;
73
+
74
+ interface GlobalWithRuntime {
75
+ [RUNTIME_GLOBAL_KEY]?: VellumPluginRuntime;
76
+ }
77
+
78
+ /**
79
+ * Install the plugin runtime bridge on `globalThis`. Idempotent — repeat
80
+ * calls are no-ops, so it's safe to invoke from tests that also touch the
81
+ * lifecycle path.
82
+ *
83
+ * Must be called BEFORE {@link loadUserPlugins} runs, otherwise plugins that
84
+ * touch `globalThis.__vellumPluginRuntime` in their module body will throw.
85
+ */
86
+ export function installPluginRuntime(): void {
87
+ const g = globalThis as GlobalWithRuntime;
88
+ if (g[RUNTIME_GLOBAL_KEY]) return;
89
+ g[RUNTIME_GLOBAL_KEY] = {
90
+ version: 1,
91
+ registerPlugin,
92
+ assistantEventHub,
93
+ getSecureKeyAsync,
94
+ };
95
+ }
96
+
97
+ /**
98
+ * Read the installed runtime. Returns `undefined` if {@link installPluginRuntime}
99
+ * hasn't been called yet — plugins should treat this as a fatal error.
100
+ *
101
+ * Exposed mainly for tests and for the rare in-process consumer that wants
102
+ * to read the bridge from the same module graph that installed it.
103
+ */
104
+ export function getPluginRuntime(): VellumPluginRuntime | undefined {
105
+ return (globalThis as GlobalWithRuntime)[RUNTIME_GLOBAL_KEY];
106
+ }
107
+
108
+ /**
109
+ * Tear down the runtime handle. Test-only — production code never uninstalls
110
+ * because the runtime lifetime is the daemon lifetime.
111
+ */
112
+ export function uninstallPluginRuntimeForTests(): void {
113
+ delete (globalThis as GlobalWithRuntime)[RUNTIME_GLOBAL_KEY];
114
+ }
@@ -11,6 +11,8 @@ import { join } from "node:path";
11
11
  import { getIsContainerized } from "../config/env-registry.js";
12
12
  import type { ChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
13
13
  import type { TrustContext } from "../daemon/trust-context.js";
14
+ import { markActivationSession } from "../memory/activation-session-store.js";
15
+ import { ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE } from "../telemetry/activation-funnel.js";
14
16
  import type { OnboardingContext } from "../types/onboarding-context.js";
15
17
  import { resolveBundledDir } from "../util/bundled-asset.js";
16
18
  import { getLogger } from "../util/logger.js";
@@ -206,7 +208,7 @@ export function ensurePromptFiles(): void {
206
208
  * overwrite the workspace file with the specified variant. No-op when
207
209
  * BOOTSTRAP.md has been deleted, modified, or the template file is missing.
208
210
  */
209
- export function maybeReseedBootstrap(templateFileName: string): void {
211
+ export function maybeReseedBootstrap(templateFileName: string): boolean {
210
212
  // Path traversal guard: reject filenames containing directory separators or
211
213
  // parent-directory references, and require a `.md` extension.
212
214
  if (
@@ -218,11 +220,11 @@ export function maybeReseedBootstrap(templateFileName: string): void {
218
220
  { templateFileName },
219
221
  "Rejected bootstrap template filename: invalid characters or extension",
220
222
  );
221
- return;
223
+ return false;
222
224
  }
223
225
 
224
226
  const bootstrapPath = getWorkspacePromptPath("BOOTSTRAP.md");
225
- if (!existsSync(bootstrapPath)) return;
227
+ if (!existsSync(bootstrapPath)) return false;
226
228
 
227
229
  const currentContent = readPromptFile(bootstrapPath);
228
230
  // Compare against the GENERIC "BOOTSTRAP.md" template, not the specified
@@ -230,7 +232,7 @@ export function maybeReseedBootstrap(templateFileName: string): void {
230
232
  // template, so this guard returns false on subsequent calls — making the
231
233
  // swap idempotent. Do NOT change the comparison target to the provided
232
234
  // template filename; that would re-swap on every prompt build.
233
- if (!isTemplateContent(currentContent, "BOOTSTRAP.md")) return;
235
+ if (!isTemplateContent(currentContent, "BOOTSTRAP.md")) return false;
234
236
 
235
237
  const templatesDir = resolveBundledDir(
236
238
  import.meta.dirname ?? __dirname,
@@ -243,7 +245,7 @@ export function maybeReseedBootstrap(templateFileName: string): void {
243
245
  { templateFileName },
244
246
  "Bootstrap template not found, keeping generic BOOTSTRAP.md",
245
247
  );
246
- return;
248
+ return false;
247
249
  }
248
250
 
249
251
  try {
@@ -253,11 +255,51 @@ export function maybeReseedBootstrap(templateFileName: string): void {
253
255
  { templateFileName },
254
256
  "Replaced generic BOOTSTRAP.md with specified template",
255
257
  );
258
+ return true;
256
259
  } catch (err) {
257
260
  log.warn(
258
261
  { err, templateFileName },
259
262
  "Failed to reseed BOOTSTRAP.md with template",
260
263
  );
264
+ return false;
265
+ }
266
+ }
267
+
268
+ /**
269
+ * Reseed BOOTSTRAP.md from the requested onboarding template (one-shot,
270
+ * idempotent) and, when that template is the activation rail AND the rail is
271
+ * now the active bootstrap, mark the conversation as an activation session.
272
+ *
273
+ * Marking happens here — at the single point where the bootstrap selection is
274
+ * known — so it lands BEFORE the agent loop resolves tools and BEFORE the model
275
+ * can call the emit tool on the first activation-rail turn. (`resolveTools`
276
+ * runs before `resolveSystemPrompt` in the loop, so the system-prompt build is
277
+ * too late to be the *only* marking site.) The marker write is best-effort and
278
+ * idempotent (`markActivationSession` swallows errors and dedups on the PK), so
279
+ * calling this from both `setOnboardingContext` and `buildSystemPrompt` is safe.
280
+ *
281
+ * The activation mark is gated on the rail actually being the active bootstrap —
282
+ * either this call just installed it, OR BOOTSTRAP.md already holds the
283
+ * activation-rail template. When the reseed no-ops because BOOTSTRAP.md is
284
+ * missing or customized to something else, the rail is NOT active, so we must
285
+ * NOT mark (otherwise non-rail conversations would pollute activation
286
+ * telemetry).
287
+ */
288
+ export function applyBootstrapTemplate(
289
+ bootstrapTemplate: string,
290
+ conversationId?: string,
291
+ ): void {
292
+ const installedActivationRail = maybeReseedBootstrap(bootstrapTemplate);
293
+ if (
294
+ bootstrapTemplate === ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE &&
295
+ conversationId &&
296
+ (installedActivationRail ||
297
+ isTemplateContent(
298
+ readPromptFile(getWorkspacePromptPath("BOOTSTRAP.md")),
299
+ ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE,
300
+ ))
301
+ ) {
302
+ markActivationSession(conversationId);
261
303
  }
262
304
  }
263
305
 
@@ -268,6 +310,13 @@ export interface BuildSystemPromptOptions {
268
310
  trustContext?: TrustContext;
269
311
  channelCapabilities?: ChannelCapabilities;
270
312
  onboardingContext?: OnboardingContext;
313
+ /**
314
+ * Conversation this prompt is being built for. Optional because several
315
+ * callers build a prompt outside a conversation (e.g. home greeting,
316
+ * suggested prompts). When present and the activation-rail bootstrap template
317
+ * is selected, the conversation is marked as an activation session.
318
+ */
319
+ conversationId?: string;
271
320
  }
272
321
 
273
322
  /**
@@ -278,11 +327,13 @@ export interface BuildSystemPromptOptions {
278
327
  * file-backed bodies, and runtime-computed transforms.
279
328
  */
280
329
  export function buildSystemPrompt(options?: BuildSystemPromptOptions): string {
281
- // One-shot bootstrap swap: if the onboarding context specifies a bootstrap
282
- // template and BOOTSTRAP.md is still the generic template, replace it with
283
- // the specified variant before the prompt reads the file.
284
- if (options?.onboardingContext?.bootstrapTemplate) {
285
- maybeReseedBootstrap(options.onboardingContext.bootstrapTemplate);
330
+ // One-shot bootstrap reseed + activation-rail marking. The marking also runs
331
+ // earlier, at `setOnboardingContext`, so the activation session is recorded
332
+ // before the agent loop resolves tools on the first turn; this call is a
333
+ // harmless idempotent backstop for prompt builds outside that path.
334
+ const bootstrapTemplate = options?.onboardingContext?.bootstrapTemplate;
335
+ if (bootstrapTemplate) {
336
+ applyBootstrapTemplate(bootstrapTemplate, options?.conversationId);
286
337
  }
287
338
 
288
339
  // Slugs used by the persona sections (`10-user-persona`,