@vellumai/assistant 0.10.9-staging.1 → 0.10.9-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/AGENTS.md +4 -0
  2. package/knip.json +1 -4
  3. package/openapi.yaml +28 -18
  4. package/package.json +1 -1
  5. package/src/__tests__/always-loaded-tools-guard.test.ts +4 -6
  6. package/src/__tests__/app-routes-csp.test.ts +13 -4
  7. package/src/__tests__/conversation-tool-setup-tools-disabled.test.ts +4 -8
  8. package/src/__tests__/disk-pressure-tools.test.ts +4 -4
  9. package/src/__tests__/file-list-tool.test.ts +22 -12
  10. package/src/__tests__/list-all-apps.test.ts +204 -0
  11. package/src/__tests__/messaging-send-tool.test.ts +67 -0
  12. package/src/__tests__/mtime-cache.test.ts +42 -0
  13. package/src/__tests__/oauth-provider-profiles.test.ts +3 -2
  14. package/src/__tests__/outlook-messaging-provider.test.ts +71 -0
  15. package/src/__tests__/plugin-app-serve-routes.test.ts +163 -0
  16. package/src/__tests__/plugin-import-boundary-guard.test.ts +1 -0
  17. package/src/__tests__/scaffold-managed-skill-tool.test.ts +47 -0
  18. package/src/__tests__/subagent-notify-parent.test.ts +4 -3
  19. package/src/__tests__/subagent-tool-filtering.test.ts +21 -21
  20. package/src/__tests__/subagent-tool-gate-mode.test.ts +6 -6
  21. package/src/apps/app-store.ts +302 -2
  22. package/src/cli/AGENTS.md +17 -0
  23. package/src/cli/commands/__tests__/gateway.test.ts +55 -0
  24. package/src/cli/commands/apps.help.ts +17 -11
  25. package/src/cli/commands/apps.ts +5 -15
  26. package/src/cli/commands/gateway.help.ts +30 -0
  27. package/src/cli/commands/gateway.ts +40 -1
  28. package/src/cli/commands/platform/__tests__/status.test.ts +0 -90
  29. package/src/cli/commands/platform/index.help.ts +3 -2
  30. package/src/cli/commands/platform/index.ts +0 -9
  31. package/src/cli/commands/routes.help.ts +13 -7
  32. package/src/cli/commands/routes.ts +5 -3
  33. package/src/cli/commands/status.help.ts +24 -0
  34. package/src/cli/commands/status.ts +29 -0
  35. package/src/cli/lib/__tests__/global-json-option.test.ts +97 -0
  36. package/src/cli/lib/__tests__/plugin-catalog-local.test.ts +1 -0
  37. package/src/cli/lib/__tests__/plugin-fingerprint.test.ts +31 -0
  38. package/src/cli/lib/bundled-marketplace.json +187 -173
  39. package/src/cli/lib/global-json-option.ts +41 -0
  40. package/src/cli/lib/plugin-fingerprint.ts +18 -7
  41. package/src/cli/program.ts +5 -0
  42. package/src/config/bundled-skills/messaging/TOOLS.json +2 -2
  43. package/src/config/bundled-skills/messaging/tools/messaging-send.ts +27 -19
  44. package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +3 -3
  45. package/src/config/feature-flag-registry.json +1 -9
  46. package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +3 -6
  47. package/src/daemon/__tests__/conversation-tool-setup-plugin-scope.test.ts +3 -6
  48. package/src/daemon/__tests__/conversation-tool-setup.test.ts +8 -5
  49. package/src/daemon/conversation-tool-setup.ts +5 -90
  50. package/src/daemon/conversation.ts +2 -2
  51. package/src/messaging/provider-types.ts +9 -0
  52. package/src/messaging/providers/outlook/adapter.ts +17 -2
  53. package/src/messaging/providers/outlook/client.ts +11 -2
  54. package/src/messaging/providers/outlook/types.ts +9 -0
  55. package/src/monitoring/__tests__/plugin-source-watch.test.ts +37 -0
  56. package/src/monitoring/plugin-source-watch.ts +53 -0
  57. package/src/oauth/seed-providers.ts +6 -0
  58. package/src/persistence/embeddings/__tests__/messages-lexical-index.test.ts +3 -0
  59. package/src/persistence/embeddings/messages-lexical-index.ts +11 -0
  60. package/src/plugin-api/conversation-turn.ts +234 -0
  61. package/src/plugin-api/index.ts +12 -0
  62. package/src/plugins/__tests__/source-fingerprint.test.ts +41 -0
  63. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +48 -0
  64. package/src/plugins/defaults/memory/__tests__/memory-retrospective-skill-card.test.ts +46 -5
  65. package/src/plugins/defaults/memory/graph/tool-handlers.ts +2 -2
  66. package/src/plugins/defaults/memory/graph-topology/__tests__/build-memory-graph.test.ts +81 -2
  67. package/src/plugins/defaults/memory/graph-topology/build-memory-graph.ts +80 -33
  68. package/src/plugins/defaults/memory/memory-retrospective-job.ts +40 -1
  69. package/src/plugins/defaults/memory/memory-retrospective-skill-card.ts +28 -2
  70. package/src/plugins/mtime-cache.ts +26 -13
  71. package/src/plugins/plugin-tree-walk.ts +30 -0
  72. package/src/plugins/source-fingerprint.ts +9 -2
  73. package/src/plugins/surface-import.ts +28 -0
  74. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +1 -1
  75. package/src/providers/speech-to-text/__tests__/vellum-speech-relay-connection.test.ts +4 -1
  76. package/src/providers/speech-to-text/vellum-speech-relay-connection.ts +5 -0
  77. package/src/runtime/routes/__tests__/gateway-status-routes.test.ts +84 -0
  78. package/src/runtime/routes/__tests__/plugins-routes.test.ts +47 -3
  79. package/src/runtime/routes/__tests__/user-routes-cli.test.ts +154 -0
  80. package/src/runtime/routes/app-management-routes.ts +96 -32
  81. package/src/runtime/routes/app-routes.ts +33 -17
  82. package/src/runtime/routes/gateway-status-routes.ts +69 -0
  83. package/src/runtime/routes/index.ts +2 -0
  84. package/src/runtime/routes/platform-routes.ts +4 -14
  85. package/src/runtime/routes/plugins-routes.ts +58 -1
  86. package/src/runtime/routes/settings-routes.ts +8 -6
  87. package/src/runtime/routes/user-route-dispatcher.ts +6 -86
  88. package/src/runtime/routes/user-route-resolution.ts +141 -0
  89. package/src/runtime/routes/user-routes-cli.ts +56 -35
  90. package/src/telemetry/AGENTS.md +15 -0
  91. package/src/telemetry/__tests__/telemetry-event-fixtures.ts +258 -0
  92. package/src/telemetry/telemetry-wire-source.json +1 -1
  93. package/src/telemetry/telemetry-wire-validation.test.ts +157 -0
  94. package/src/telemetry/telemetry-wire-validation.ts +125 -0
  95. package/src/telemetry/telemetry-wire.generated.ts +60 -9
  96. package/src/telemetry/types.test.ts +38 -0
  97. package/src/telemetry/types.ts +183 -91
  98. package/src/telemetry/usage-telemetry-reporter.ts +5 -0
  99. package/src/telemetry/watchdog-direct-emit.ts +6 -0
  100. package/src/tools/skills/scaffold-managed.ts +43 -11
@@ -4,7 +4,10 @@ import type { PageIndexEntry } from "../../v2/page-index.js";
4
4
  import type { Slug } from "../../v3/types.js";
5
5
  import { assembleMemoryGraph } from "../build-memory-graph.js";
6
6
 
7
- function entry(slug: string, over: Partial<PageIndexEntry> = {}): PageIndexEntry {
7
+ function entry(
8
+ slug: string,
9
+ over: Partial<PageIndexEntry> = {},
10
+ ): PageIndexEntry {
8
11
  return {
9
12
  id: 0,
10
13
  slug,
@@ -40,7 +43,9 @@ describe("assembleMemoryGraph", () => {
40
43
  entry("skills/agent-mail", { modifiedAt: 0 }),
41
44
  entry("send-email", { modifiedAt: 0 }),
42
45
  ],
43
- staticAdjacency: adjacency([["my-concept", "skills/agent-mail", undefined]]),
46
+ staticAdjacency: adjacency([
47
+ ["my-concept", "skills/agent-mail", undefined],
48
+ ]),
44
49
  });
45
50
 
46
51
  const byId = new Map(nodes.map((n) => [n.id, n]));
@@ -127,4 +132,78 @@ describe("assembleMemoryGraph", () => {
127
132
  expect(nodes.some((n) => n.id === edge.target)).toBe(true);
128
133
  }
129
134
  });
135
+
136
+ it("prunes disconnected functionality nodes but keeps connected ones and all concepts", () => {
137
+ const { nodes } = assembleMemoryGraph({
138
+ entries: [
139
+ entry("lonely-concept"), // concept, degree 0 → kept
140
+ entry("linked-concept"), // concept, links to a skill
141
+ entry("skills/connected", { modifiedAt: 0 }), // skill, degree 1 → kept
142
+ entry("skills/orphan", { modifiedAt: 0 }), // skill, degree 0 → pruned
143
+ entry("cli-commands/orphan", { modifiedAt: 0 }), // capability, degree 0 → pruned
144
+ ],
145
+ staticAdjacency: adjacency([
146
+ ["linked-concept", "skills/connected", undefined],
147
+ ]),
148
+ pruneDisconnectedNonConcepts: true,
149
+ });
150
+
151
+ expect(nodes.map((n) => n.id).sort()).toEqual([
152
+ "linked-concept",
153
+ "lonely-concept",
154
+ "skills/connected",
155
+ ]);
156
+ });
157
+
158
+ it("keeps disconnected functionality nodes when pruning is off (default)", () => {
159
+ const { nodes } = assembleMemoryGraph({
160
+ entries: [entry("a"), entry("skills/orphan", { modifiedAt: 0 })],
161
+ staticAdjacency: adjacency([]),
162
+ });
163
+ expect(nodes.map((n) => n.id).sort()).toEqual(["a", "skills/orphan"]);
164
+ });
165
+
166
+ it("re-prunes a functionality node stranded by truncation", () => {
167
+ const { nodes, edges, truncated } = assembleMemoryGraph({
168
+ entries: [
169
+ entry("hub"),
170
+ entry("h1"),
171
+ entry("h2"),
172
+ entry("h3"),
173
+ entry("p"),
174
+ entry("q"),
175
+ entry("skills/s", { modifiedAt: 0 }),
176
+ ],
177
+ // hub has degree 3; skills/s has degree 2 (p, q). The cap keeps the top 2
178
+ // by degree (hub, skills/s) but drops every one of skills/s's neighbors.
179
+ staticAdjacency: adjacency([
180
+ ["hub", "h1", undefined],
181
+ ["hub", "h2", undefined],
182
+ ["hub", "h3", undefined],
183
+ ["skills/s", "p", undefined],
184
+ ["skills/s", "q", undefined],
185
+ ]),
186
+ maxNodes: 2,
187
+ pruneDisconnectedNonConcepts: true,
188
+ });
189
+
190
+ expect(truncated).toBe(true);
191
+ // skills/s survived the cap but lost both neighbors → re-pruned as isolated;
192
+ // the isolated concept hub is kept (concepts always survive).
193
+ expect(nodes.map((n) => n.id)).toEqual(["hub"]);
194
+ expect(edges).toEqual([]);
195
+ });
196
+
197
+ it("treats a real page under a reserved prefix as a concept, not a prunable skill", () => {
198
+ // A non-colliding user page like `skills/my-notes` survives the page index
199
+ // with a real mtime; it must classify as a concept (by modifiedAt, not the
200
+ // slug prefix) and therefore never be pruned as a disconnected skill.
201
+ const { nodes } = assembleMemoryGraph({
202
+ entries: [entry("skills/my-notes", { modifiedAt: 5 })],
203
+ staticAdjacency: adjacency([]),
204
+ pruneDisconnectedNonConcepts: true,
205
+ });
206
+ expect(nodes).toHaveLength(1);
207
+ expect(nodes[0]).toMatchObject({ id: "skills/my-notes", kind: "concept" });
208
+ });
130
209
  });
@@ -22,7 +22,10 @@ import { getWorkspaceDir } from "../paths.js";
22
22
  import { getPageIndex, type PageIndexEntry } from "../v2/page-index.js";
23
23
  import { readPage, renderPageContent } from "../v2/page-store.js";
24
24
  import { isSkillSlug } from "../v2/skill-store.js";
25
- import { isCapabilitySlug } from "../v3/capabilities.js";
25
+ import {
26
+ isCapabilitySlug,
27
+ renderCapabilityContent,
28
+ } from "../v3/capabilities.js";
26
29
  import { buildEdgeGraph } from "../v3/edge.js";
27
30
  import { computeLearnedEdgeGraph } from "../v3/learned-edges.js";
28
31
  import type { Slug } from "../v3/types.js";
@@ -78,24 +81,18 @@ function humanizeSlug(slug: string): string {
78
81
  return words.replace(/\b\w/g, (c) => c.toUpperCase());
79
82
  }
80
83
 
81
- /** Node taxonomy tag used for coloring. Synthetic capability slugs (skills /
82
- * CLI commands) carry `modifiedAt: 0`; real concept pages carry a file mtime. */
84
+ /** Node taxonomy tag used for coloring. Only synthetic rows (`modifiedAt: 0`)
85
+ * are functionality: skills carry the `skills/` prefix, other synthetics (CLI
86
+ * commands) are capabilities. A real on-disk page keeps a file mtime and is a
87
+ * concept even when it happens to sit under a reserved prefix (e.g. a user page
88
+ * `skills/my-notes` with no matching skill survives the page index). */
83
89
  function nodeKind(entry: PageIndexEntry): string {
84
- if (isSkillSlug(entry.slug)) {
85
- return "skill";
86
- }
87
90
  if (entry.modifiedAt <= 0) {
88
- return "capability";
91
+ return isSkillSlug(entry.slug) ? "skill" : "capability";
89
92
  }
90
93
  return "concept";
91
94
  }
92
95
 
93
- /** A real concept page: on-disk (has an mtime) and not a synthetic skill or
94
- * CLI-command capability slug. The graph shows concepts only. */
95
- function isConceptEntry(entry: PageIndexEntry): boolean {
96
- return entry.modifiedAt > 0 && !isCapabilitySlug(entry.slug);
97
- }
98
-
99
96
  export interface AssembleMemoryGraphInput {
100
97
  /** Every article node in the corpus (page-index entries). */
101
98
  entries: readonly PageIndexEntry[];
@@ -105,6 +102,12 @@ export interface AssembleMemoryGraphInput {
105
102
  learnedAdjacency?: Adjacency;
106
103
  /** Node cap; defaults to {@link DEFAULT_MAX_NODES}. */
107
104
  maxNodes?: number;
105
+ /**
106
+ * When set, drop functionality nodes (kind `skill` / `capability`) that ended
107
+ * up with no edges — a skill nobody links to or co-selects is inert clutter.
108
+ * Concept nodes are always kept, even when isolated.
109
+ */
110
+ pruneDisconnectedNonConcepts?: boolean;
108
111
  }
109
112
 
110
113
  /**
@@ -199,6 +202,12 @@ export function assembleMemoryGraph(input: AssembleMemoryGraphInput): {
199
202
  return node;
200
203
  });
201
204
 
205
+ // Prune disconnected functionality nodes (see the option's doc). `weight` is
206
+ // the node's degree, so weight 0 ⇒ no incident edges ⇒ no edge cleanup needed.
207
+ if (input.pruneDisconnectedNonConcepts) {
208
+ nodes = nodes.filter((n) => n.kind === "concept" || (n.weight ?? 0) > 0);
209
+ }
210
+
202
211
  if (nodes.length <= maxNodes) {
203
212
  return { nodes, edges };
204
213
  }
@@ -211,6 +220,20 @@ export function assembleMemoryGraph(input: AssembleMemoryGraphInput): {
211
220
  const keptEdges = edges.filter(
212
221
  (e) => kept.has(e.source) && kept.has(e.target),
213
222
  );
223
+
224
+ // Truncation can strand a functionality node whose only neighbors were
225
+ // dropped. Re-prune against post-truncation degree so the connected-only
226
+ // guarantee holds even in a capped graph. An isolated node touches no kept
227
+ // edge by definition, so no further edge cleanup is needed.
228
+ if (input.pruneDisconnectedNonConcepts) {
229
+ const connected = new Set<string>();
230
+ for (const e of keptEdges) {
231
+ connected.add(e.source);
232
+ connected.add(e.target);
233
+ }
234
+ nodes = nodes.filter((n) => n.kind === "concept" || connected.has(n.id));
235
+ }
236
+
214
237
  return { nodes, edges: keptEdges, truncated: true };
215
238
  }
216
239
 
@@ -228,15 +251,28 @@ export async function getMemoryGraph(
228
251
 
229
252
  const workspaceDir = getWorkspaceDir();
230
253
  const pageIndex = await getPageIndex(workspaceDir);
231
- // Concepts only: exclude synthetic skill / CLI-command slugs so the graph is
232
- // purely the assistant's learned/authored concept pages. Edges to excluded
233
- // slugs drop out downstream because they aren't in the node set.
234
- const conceptEntries = pageIndex.entries.filter(isConceptEntry);
254
+ // Concepts plus functionality (skills / CLI-command capabilities). Feeding the
255
+ // full set to the edge builders is what lets a concept's `[[skills/foo]]` link
256
+ // and skill↔concept co-selections resolve to real edges buildEdgeGraph and
257
+ // computeLearnedEdgeGraph both drop endpoints outside the set they're given.
258
+ // Functionality nodes that end up disconnected are pruned in assembleMemoryGraph.
259
+ const entries = pageIndex.entries;
260
+
261
+ // Synthetic rows (skills / CLI commands) carry `modifiedAt: 0` and have no
262
+ // on-disk page. Keyed by slug (not prefix) so a real user page that happens
263
+ // to live under a reserved prefix is NOT mistaken for a synthetic one.
264
+ const syntheticSlugs = new Set(
265
+ entries.filter((e) => e.modifiedAt <= 0).map((e) => e.slug),
266
+ );
235
267
 
236
268
  // Raw (frontmatter + body) page reader, matching the v3 lane build. A read
237
269
  // that rejects drops that article's authored/wikilink edges but keeps its
238
- // numeric fallbacks.
270
+ // numeric fallbacks. Synthetic capability rows have no page, so short-circuit
271
+ // their guaranteed-miss read; a real page is read so its links are captured.
239
272
  const pageRaw = async (slug: Slug): Promise<string> => {
273
+ if (syntheticSlugs.has(slug)) {
274
+ return "";
275
+ }
240
276
  const page = await readPage(workspaceDir, slug);
241
277
  if (!page) {
242
278
  throw new Error(`page not found: ${slug}`);
@@ -244,7 +280,7 @@ export async function getMemoryGraph(
244
280
  return renderPageContent(page);
245
281
  };
246
282
 
247
- const staticGraph = await buildEdgeGraph(conceptEntries, pageRaw, {
283
+ const staticGraph = await buildEdgeGraph(entries, pageRaw, {
248
284
  hubDegree: config.memory.v3.edge.hubDegree,
249
285
  });
250
286
 
@@ -265,39 +301,50 @@ export async function getMemoryGraph(
265
301
  maxPerPage: Math.max(learned.maxPerPage, GRAPH_LEARNED_MIN_MAX_PER_PAGE),
266
302
  now: Date.now(),
267
303
  windowMs: LEARNED_EDGES_WINDOW_DAYS * DAY_MS,
268
- knownSlugs: new Set(conceptEntries.map((e) => e.slug)),
304
+ knownSlugs: new Set(entries.map((e) => e.slug)),
269
305
  },
270
306
  );
271
307
 
272
308
  const assembled = assembleMemoryGraph({
273
- entries: conceptEntries,
309
+ entries,
274
310
  staticAdjacency: staticGraph.adjacency,
275
311
  learnedAdjacency: learnedGraph.adjacency,
312
+ pruneDisconnectedNonConcepts: true,
276
313
  });
277
314
 
278
315
  return { backend: BACKEND_MEMORY_V3, supported: true, ...assembled };
279
316
  }
280
317
 
281
318
  /**
282
- * Fetch a single concept node's content (its markdown body) by id. Used when a
283
- * user opens a node in the graph. Concepts only — skill/capability slugs and
284
- * unreadable pages return `{ found: false }`.
319
+ * Fetch a single node's content by id. Used when a user opens a node in the
320
+ * graph. Concept nodes return their page's markdown body; functionality nodes
321
+ * (skills / CLI commands) return the rendered capability statement. Unknown or
322
+ * unreadable ids return `{ found: false }`.
285
323
  */
286
324
  export async function getMemoryGraphNode(
287
325
  config: AssistantConfig,
288
326
  id: string,
289
327
  ): Promise<MemoryGraphNodeDetail> {
290
- if (
291
- !isMemoryConceptGraphEnabled(config) ||
292
- !isMemoryV3Live(config) ||
293
- !id ||
294
- isCapabilitySlug(id)
295
- ) {
328
+ if (!isMemoryConceptGraphEnabled(config) || !isMemoryV3Live(config) || !id) {
296
329
  return { found: false };
297
330
  }
331
+ // Seeded skill/CLI capabilities take precedence over any on-disk page at the
332
+ // same slug: the page index drops a colliding page and lets the synthetic win
333
+ // (v2/page-index.ts), so a `skills/foo` node built as the capability must not
334
+ // surface a stale disk page. renderCapabilityContent returns the rendered
335
+ // statement for a seeded capability, "" for an unseeded reserved-prefix slug
336
+ // (a real user page), and null for a normal concept slug.
337
+ if (isCapabilitySlug(id)) {
338
+ const content = renderCapabilityContent(id);
339
+ if (content) {
340
+ return { found: true, title: humanizeSlug(id), content };
341
+ }
342
+ }
343
+ // A real on-disk page: a concept, or a user page under a reserved prefix that
344
+ // isn't a seeded capability (kept in the index with a real mtime).
298
345
  const page = await readPage(getWorkspaceDir(), id).catch(() => null);
299
- if (!page) {
300
- return { found: false };
346
+ if (page) {
347
+ return { found: true, title: humanizeSlug(id), content: page.body };
301
348
  }
302
- return { found: true, title: humanizeSlug(id), content: page.body };
349
+ return { found: false };
303
350
  }
@@ -77,6 +77,7 @@ import {
77
77
  import { resolveUserSlug } from "../../../prompts/persona-resolver.js";
78
78
  import type { SystemPromptPersonaOverride } from "../../../prompts/system-prompt.js";
79
79
  import { wakeAgentForOpportunity } from "../../../runtime/agent-wake.js";
80
+ import { recordWatchdogEvent } from "../../../telemetry/watchdog-events-store.js";
80
81
  import { findMostRecentRetrospectiveFor } from "./find-most-recent-retrospective-for.js";
81
82
  import { getLogger } from "./logging.js";
82
83
  import { getRetrospectiveMessagesAfter } from "./memory-retrospective-accounting.js";
@@ -115,6 +116,9 @@ const FOLLOW_UP_JOB_TYPES: readonly MemoryJobType[] = [] as const;
115
116
  */
116
117
  export const SOURCE_PROCESSING_REQUEUE_DELAY_MS = 60_000;
117
118
 
119
+ /** Watchdog check_name for the per-run retrospective outcome counter. */
120
+ const MEMORY_RETROSPECTIVE_RUN_CHECK_NAME = "memory_retrospective_run";
121
+
118
122
  export type MemoryRetrospectiveOutcome =
119
123
  | { kind: "disabled" }
120
124
  | { kind: "no_new_messages" }
@@ -138,7 +142,42 @@ export async function memoryRetrospectiveJob(
138
142
  return { kind: "no_new_messages" };
139
143
  }
140
144
 
141
- return runForkBasedRetrospective(sourceConversationId, config);
145
+ // Central health counter (admin analytics groups on the watchdog
146
+ // check_name): one event per run with its outcome kind. A run that
147
+ // throws records outcome "error" before the exception continues to the
148
+ // jobs worker's retry machinery, so a fleet-wide spike in
149
+ // `wake_failed`/`error` (e.g. a provider outage on the retrospective's
150
+ // resolved model) is visible without log access. The emitter itself
151
+ // never throws — the run's outcome must reach the jobs worker
152
+ // regardless.
153
+ const emitRunOutcome = (outcome: string, reason?: string): void => {
154
+ try {
155
+ recordWatchdogEvent({
156
+ checkName: MEMORY_RETROSPECTIVE_RUN_CHECK_NAME,
157
+ value: 1,
158
+ detail: {
159
+ outcome,
160
+ ...(reason ? { reason: reason.slice(0, 200) } : {}),
161
+ },
162
+ });
163
+ } catch {
164
+ // recordWatchdogEvent already no-ops on opt-out and a missing
165
+ // telemetry DB; anything past that is not worth surfacing here.
166
+ }
167
+ };
168
+
169
+ let outcome: MemoryRetrospectiveOutcome;
170
+ try {
171
+ outcome = await runForkBasedRetrospective(sourceConversationId, config);
172
+ } catch (err) {
173
+ emitRunOutcome("error", err instanceof Error ? err.message : String(err));
174
+ throw err;
175
+ }
176
+ emitRunOutcome(
177
+ outcome.kind,
178
+ outcome.kind === "wake_failed" ? outcome.reason : undefined,
179
+ );
180
+ return outcome;
142
181
  }
143
182
 
144
183
  // ---------------------------------------------------------------------------
@@ -56,11 +56,15 @@ import {
56
56
  upsertSkillCardInsertJob,
57
57
  } from "../../../persistence/jobs-store.js";
58
58
  import { publishConversationMessagesChanged } from "../../../runtime/sync/resource-sync-events.js";
59
+ import { recordWatchdogEvent } from "../../../telemetry/watchdog-events-store.js";
59
60
  import { getLogger } from "./logging.js";
60
61
  import { SKILL_CARD_MESSAGE_KIND } from "./memory-retrospective-constants.js";
61
62
 
62
63
  const log = getLogger("memory-retrospective-skill-card");
63
64
 
65
+ /** Watchdog check_name for the per-card delivery counter. */
66
+ const SKILL_CARD_DELIVERED_CHECK_NAME = "skill_card_delivered";
67
+
64
68
  /**
65
69
  * A skill authored by a retrospective run, as recorded by the scaffold
66
70
  * executor at creation time. All values are the executor's post-normalization
@@ -277,11 +281,19 @@ async function insertOrDeferSkillCard(
277
281
  return;
278
282
  }
279
283
  const surfaceId = `skill-card-${runConversationId}`;
284
+ // Copy matches the web card's row sentence (skill-created-card.tsx): the
285
+ // card reads as the assistant sharing what it picked up, not a technical
286
+ // "skill created" notice. The title covers renderers that only show the
287
+ // block title; the web card renders per-skill rows from `data.skills`.
288
+ const learnedSentence =
289
+ skills.length === 1
290
+ ? `I just learned how to do ${skills[0]!.name}`
291
+ : `I just learned how to do ${skills.length} new things`;
280
292
  const surfaceBlock = {
281
293
  type: "ui_surface",
282
294
  surfaceId,
283
295
  surfaceType: "skill_card",
284
- title: "New skill learned",
296
+ title: learnedSentence,
285
297
  display: "inline",
286
298
  data: {
287
299
  skills: skills.map((s) => ({
@@ -298,7 +310,7 @@ async function insertOrDeferSkillCard(
298
310
  // rendered content blocks so the card is never double-rendered.
299
311
  const fallbackBlock = {
300
312
  type: "text",
301
- text: `New skill learned: ${skills.map((s) => s.name).join(", ")}`,
313
+ text: `I just learned how to do ${skills.map((s) => s.name).join(", ")}`,
302
314
  _surfaceFallback: true,
303
315
  };
304
316
  const persisted = await addMessage(
@@ -334,6 +346,20 @@ async function insertOrDeferSkillCard(
334
346
  );
335
347
  }
336
348
  publishConversationMessagesChanged(sourceConversationId);
349
+ // Central delivery counter (admin analytics groups on the watchdog
350
+ // check_name): one event per delivered card, value = skills on it. The
351
+ // dedup early-return above keeps retried deliveries from double-counting.
352
+ // Never throws — the card is already inserted.
353
+ try {
354
+ recordWatchdogEvent({
355
+ checkName: SKILL_CARD_DELIVERED_CHECK_NAME,
356
+ value: skills.length,
357
+ detail: { skill_count: skills.length },
358
+ });
359
+ } catch {
360
+ // recordWatchdogEvent already no-ops on opt-out and a missing telemetry
361
+ // DB; anything past that is not worth surfacing here.
362
+ }
337
363
  log.info(
338
364
  {
339
365
  sourceConversationId,
@@ -72,6 +72,7 @@ import {
72
72
  import {
73
73
  clearSurfaceImportInflight,
74
74
  evictModule,
75
+ getFileSignature,
75
76
  getMtime,
76
77
  importWithTimeout,
77
78
  setSurfaceImportTimeout,
@@ -204,8 +205,13 @@ const disabledPluginDirs = new Set<string>();
204
205
  */
205
206
  let lastVersions: Record<string, PluginSourceVersion> = {};
206
207
 
207
- /** mtime of the sentinel document as of the last look; 0 = never seen. */
208
- let lastSentinelMtime = 0;
208
+ /**
209
+ * Change signature (`mtimeMs:size:ino`) of the sentinel document as of the last
210
+ * look; `""` = never seen. A composite signature rather than mtime alone so a
211
+ * publish is detected even when the filesystem's mtime granularity can't move
212
+ * the timestamp between two rewrites (see {@link getFileSignature}).
213
+ */
214
+ let lastSentinelSignature = "";
209
215
 
210
216
  /** In-flight reconcile — concurrent dispatches await it rather than racing. */
211
217
  let reconcileInFlight: Promise<void> | null = null;
@@ -254,11 +260,18 @@ export async function getUserHooksFor<TCtx = unknown>(
254
260
 
255
261
  /**
256
262
  * Dispatch-path gate: one stat of the source-versions sentinel. When its
257
- * mtime is unchanged since the last look (the overwhelmingly common case)
258
- * this returns immediately, and dispatch runs entirely on memory. When the
259
- * watcher published a change, the document is applied before the dispatch
263
+ * change signature is unchanged since the last look (the overwhelmingly common
264
+ * case) this returns immediately, and dispatch runs entirely on memory. When
265
+ * the watcher published a change, the document is applied before the dispatch
260
266
  * proceeds, so the turn that follows an edit already runs the new code.
261
267
  *
268
+ * The signature is `mtimeMs:size:ino`, not mtime alone: the sentinel is
269
+ * published via temp-file + atomic rename, which swaps in a fresh inode on
270
+ * every write, so a runtime install is picked up even on filesystems whose
271
+ * mtime granularity is too coarse to move the timestamp between two publishes
272
+ * (virtiofs / 9p / network mounts). With an mtime-only gate such a publish is
273
+ * invisible until the next daemon restart re-walks the plugins directory.
274
+ *
262
275
  * A missing or unreadable sentinel is degraded mode, not an error: the
263
276
  * boot-time state keeps serving, and live reload resumes when the monitor
264
277
  * publishes again.
@@ -273,14 +286,14 @@ async function maybeReconcileFromSentinel(): Promise<void> {
273
286
  await reconcileInFlight;
274
287
  continue;
275
288
  }
276
- const mtime = getMtime(getSourceVersionsPath());
277
- if (mtime === lastSentinelMtime) {
289
+ const signature = getFileSignature(getSourceVersionsPath());
290
+ if (signature === lastSentinelSignature) {
278
291
  return;
279
292
  }
280
- // Claim the mtime before any async work, so a concurrent dispatch either
281
- // finds the in-flight promise above or skips on the updated mtime.
282
- lastSentinelMtime = mtime;
283
- if (mtime === 0) {
293
+ // Claim the signature before any async work, so a concurrent dispatch
294
+ // either finds the in-flight promise above or skips on the updated value.
295
+ lastSentinelSignature = signature;
296
+ if (signature === "") {
284
297
  return;
285
298
  }
286
299
  const doc = readSourceVersions();
@@ -544,7 +557,7 @@ function seedVersionBaseline(): void {
544
557
  };
545
558
  }
546
559
  lastVersions = seeded;
547
- lastSentinelMtime = getMtime(getSourceVersionsPath());
560
+ lastSentinelSignature = getFileSignature(getSourceVersionsPath());
548
561
  }
549
562
 
550
563
  // ─── Tool cache ──────────────────────────────────────────────────────────────
@@ -1031,7 +1044,7 @@ export function resetPluginCacheForTests(): void {
1031
1044
  activatedNames.clear();
1032
1045
  disabledPluginDirs.clear();
1033
1046
  lastVersions = {};
1034
- lastSentinelMtime = 0;
1047
+ lastSentinelSignature = "";
1035
1048
  reconcileInFlight = null;
1036
1049
  }
1037
1050
 
@@ -40,12 +40,39 @@ export const PRESERVED_ENTRIES = [
40
40
  ".disabled",
41
41
  ] as const;
42
42
 
43
+ /**
44
+ * A generated app build directory: `apps/<app>/dist`. This is compiled output
45
+ * (the plugin source watcher builds each multi-file app's `src/` into its
46
+ * sibling `dist/`), never tracked source, so every fingerprint walk excludes
47
+ * it:
48
+ *
49
+ * - the **live-reload** change detector (`./source-fingerprint.ts`), so the
50
+ * watcher's own compile does not read as a source change and re-trigger
51
+ * itself in a loop, and
52
+ * - the **install/drift** fingerprint (`../cli/lib/plugin-fingerprint.ts`), so
53
+ * generated output is not reported as drift/added against the pinned commit.
54
+ *
55
+ * Scoped to `apps/<app>/dist` specifically — a plugin's own top-level `dist/`
56
+ * (if it ships built code its hooks import) is still tracked.
57
+ */
58
+ export function isGeneratedAppBuildDir(relDir: string): boolean {
59
+ const parts = relDir.split("/");
60
+ return parts.length === 3 && parts[0] === "apps" && parts[2] === "dist";
61
+ }
62
+
43
63
  /** Options controlling which entries a {@link walkPluginTree} visits. */
44
64
  export interface PluginTreeWalkOptions {
45
65
  /** Top-level entry names to skip (e.g. {@link PRESERVED_ENTRIES}). */
46
66
  readonly excludeRootEntries?: Iterable<string>;
47
67
  /** Directory names skipped at any depth (e.g. `node_modules`). */
48
68
  readonly excludeDirsAnywhere?: ReadonlySet<string>;
69
+ /**
70
+ * Skip a directory (and its whole subtree) by its POSIX path relative to the
71
+ * walk root. Called for each directory before descending — use for
72
+ * path-scoped exclusions a bare directory name can't express (e.g. generated
73
+ * `apps/<app>/dist`, via {@link isGeneratedAppBuildDir}).
74
+ */
75
+ readonly excludeDir?: (relDir: string) => boolean;
49
76
  /** Skip entries whose name starts with `.`, at any depth. */
50
77
  readonly excludeDotEntries?: boolean;
51
78
  /**
@@ -96,6 +123,9 @@ export function walkPluginTree(
96
123
  if (options.excludeDirsAnywhere?.has(name) === true) {
97
124
  continue;
98
125
  }
126
+ if (options.excludeDir?.(rel) === true) {
127
+ continue;
128
+ }
99
129
  walk(rel);
100
130
  } else if (entry.isFile()) {
101
131
  visit(rel, join(absDir, name));
@@ -18,7 +18,9 @@
18
18
  *
19
19
  * Exclusions: dot-entries anywhere (`.disabled`, `.git`, …), `node_modules`
20
20
  * anywhere (vendored deps change only through install flows, which recycle
21
- * the plugin directory), and at the plugin root only — the runtime-owned
21
+ * the plugin directory), generated app build output ({@link
22
+ * isGeneratedAppBuildDir}, so the watcher's own compile does not re-trigger a
23
+ * pass), and — at the plugin root only — the runtime-owned
22
24
  * {@link PRESERVED_ENTRIES} (`data/`, `config.json`, `install-meta.json`),
23
25
  * none of which are importable source. Symlinked entries inside the tree are
24
26
  * neither watched nor followed (see {@link walkPluginTree}); a symlinked
@@ -30,7 +32,11 @@
30
32
 
31
33
  import { realpathSync, statSync } from "node:fs";
32
34
 
33
- import { PRESERVED_ENTRIES, walkPluginTree } from "./plugin-tree-walk.js";
35
+ import {
36
+ isGeneratedAppBuildDir,
37
+ PRESERVED_ENTRIES,
38
+ walkPluginTree,
39
+ } from "./plugin-tree-walk.js";
34
40
 
35
41
  /** Directory names skipped at any depth. */
36
42
  const EXCLUDED_DIRS_ANYWHERE = new Set(["node_modules"]);
@@ -78,6 +84,7 @@ export function snapshotPluginSource(pluginDir: string): SourceSnapshot {
78
84
  {
79
85
  excludeRootEntries: PRESERVED_ENTRIES,
80
86
  excludeDirsAnywhere: EXCLUDED_DIRS_ANYWHERE,
87
+ excludeDir: isGeneratedAppBuildDir,
81
88
  excludeDotEntries: true,
82
89
  bestEffort: true,
83
90
  },
@@ -50,6 +50,34 @@ export function getMtime(filePath: string): number {
50
50
  }
51
51
  }
52
52
 
53
+ /**
54
+ * Composite change signature for a file — `mtimeMs:size:ino` — or `""` when the
55
+ * file is absent or can't be stat'd.
56
+ *
57
+ * Unlike {@link getMtime}, this detects a rewrite even when the filesystem's
58
+ * mtime resolution is too coarse to move the timestamp (or two rewrites land in
59
+ * the same timestamp granule). The source-versions sentinel is published via a
60
+ * temp-file + atomic rename, which swaps in a fresh inode on every write, so
61
+ * `ino` changes even when `mtimeMs` does not — and `size` shifts too whenever
62
+ * the plugin set does. Gating the reconcile on this signature (rather than
63
+ * mtime alone) is what lets a plugin installed at runtime go live on
64
+ * filesystems whose mtime granularity is coarse (virtiofs / 9p / network mounts
65
+ * — the same mounts the plugin-source watcher polls precisely because their
66
+ * timestamps are unreliable) instead of only after the next daemon restart.
67
+ *
68
+ * Change detection only — the exact numbers are never interpreted, so inode
69
+ * values above 2^53 losing float precision cannot cause a missed update: `mtime`
70
+ * and `size` still move on a real publish.
71
+ */
72
+ export function getFileSignature(filePath: string): string {
73
+ try {
74
+ const st = statSync(filePath);
75
+ return `${st.mtimeMs}:${st.size}:${st.ino}`;
76
+ } catch {
77
+ return "";
78
+ }
79
+ }
80
+
53
81
  /**
54
82
  * Evict `filePath` from the runtime module registry so the next dynamic
55
83
  * `import()` of that path re-evaluates the file from disk.
@@ -204,7 +204,7 @@ describe("VellumManagedRealtimeTranscriber", () => {
204
204
  expect.objectContaining({
205
205
  type: "error",
206
206
  category: "auth",
207
- message: expect.stringContaining("platform connect"),
207
+ message: expect.stringContaining("VELAY_BASE_URL"),
208
208
  }),
209
209
  );
210
210
  });
@@ -54,7 +54,10 @@ describe("resolveSpeechRelayConnection", () => {
54
54
 
55
55
  describe("mapVelayError", () => {
56
56
  test("maps the relay contract's codes onto categories", () => {
57
- expect(mapVelayError({ code: "invalid_key" }).category).toBe("auth");
57
+ expect(mapVelayError({ code: "invalid_key" })).toMatchObject({
58
+ category: "auth",
59
+ message: expect.stringContaining("VELAY_BASE_URL"),
60
+ });
58
61
  expect(
59
62
  mapVelayError({ code: "missing_platform_connection" }),
60
63
  ).toMatchObject({
@@ -99,6 +99,11 @@ export function mapVelayError(error: VelayErrorInfo): {
99
99
  } {
100
100
  switch (error.code) {
101
101
  case "invalid_key":
102
+ return {
103
+ category: "auth",
104
+ message:
105
+ "The Vellum speech relay rejected this assistant's API key — the assistant's platform environment may not match the relay it is dialing (set VELAY_BASE_URL on the gateway to the matching environment, e.g. https://velay-staging.vellum.ai for staging).",
106
+ };
102
107
  case "missing_platform_connection":
103
108
  return {
104
109
  category: "auth",