@vellumai/assistant 0.3.3 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/Dockerfile +2 -0
  2. package/README.md +45 -18
  3. package/package.json +1 -1
  4. package/scripts/ipc/generate-swift.ts +13 -0
  5. package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +100 -0
  6. package/src/__tests__/approval-hardcoded-copy-guard.test.ts +41 -0
  7. package/src/__tests__/approval-message-composer.test.ts +253 -0
  8. package/src/__tests__/call-domain.test.ts +12 -2
  9. package/src/__tests__/call-orchestrator.test.ts +391 -1
  10. package/src/__tests__/call-routes-http.test.ts +27 -2
  11. package/src/__tests__/channel-approval-routes.test.ts +397 -135
  12. package/src/__tests__/channel-approvals.test.ts +99 -3
  13. package/src/__tests__/channel-delivery-store.test.ts +30 -4
  14. package/src/__tests__/channel-guardian.test.ts +261 -22
  15. package/src/__tests__/channel-readiness-service.test.ts +257 -0
  16. package/src/__tests__/config-schema.test.ts +2 -1
  17. package/src/__tests__/credential-security-invariants.test.ts +1 -0
  18. package/src/__tests__/daemon-lifecycle.test.ts +636 -0
  19. package/src/__tests__/dictation-mode-detection.test.ts +63 -0
  20. package/src/__tests__/entity-search.test.ts +615 -0
  21. package/src/__tests__/gateway-only-enforcement.test.ts +19 -13
  22. package/src/__tests__/handlers-twilio-config.test.ts +480 -0
  23. package/src/__tests__/ipc-snapshot.test.ts +63 -0
  24. package/src/__tests__/messaging-send-tool.test.ts +65 -0
  25. package/src/__tests__/run-orchestrator-assistant-events.test.ts +4 -0
  26. package/src/__tests__/run-orchestrator.test.ts +22 -0
  27. package/src/__tests__/secret-scanner.test.ts +223 -0
  28. package/src/__tests__/session-runtime-assembly.test.ts +85 -1
  29. package/src/__tests__/shell-parser-property.test.ts +357 -2
  30. package/src/__tests__/sms-messaging-provider.test.ts +125 -0
  31. package/src/__tests__/system-prompt.test.ts +25 -1
  32. package/src/__tests__/tool-executor-lifecycle-events.test.ts +34 -1
  33. package/src/__tests__/twilio-routes.test.ts +39 -3
  34. package/src/__tests__/twitter-cli-error-shaping.test.ts +2 -2
  35. package/src/__tests__/user-reference.test.ts +68 -0
  36. package/src/__tests__/web-search.test.ts +1 -1
  37. package/src/__tests__/work-item-output.test.ts +110 -0
  38. package/src/calls/call-domain.ts +8 -5
  39. package/src/calls/call-orchestrator.ts +85 -22
  40. package/src/calls/twilio-config.ts +17 -11
  41. package/src/calls/twilio-rest.ts +276 -0
  42. package/src/calls/twilio-routes.ts +39 -1
  43. package/src/cli/map.ts +6 -0
  44. package/src/commands/__tests__/cc-command-registry.test.ts +67 -0
  45. package/src/commands/cc-command-registry.ts +14 -1
  46. package/src/config/bundled-skills/claude-code/TOOLS.json +10 -3
  47. package/src/config/bundled-skills/knowledge-graph/SKILL.md +15 -0
  48. package/src/config/bundled-skills/knowledge-graph/TOOLS.json +56 -0
  49. package/src/config/bundled-skills/knowledge-graph/tools/graph-query.ts +185 -0
  50. package/src/config/bundled-skills/media-processing/SKILL.md +199 -0
  51. package/src/config/bundled-skills/media-processing/TOOLS.json +320 -0
  52. package/src/config/bundled-skills/media-processing/services/capability-registry.ts +137 -0
  53. package/src/config/bundled-skills/media-processing/services/event-detection-service.ts +280 -0
  54. package/src/config/bundled-skills/media-processing/services/feedback-aggregation.ts +144 -0
  55. package/src/config/bundled-skills/media-processing/services/feedback-store.ts +136 -0
  56. package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +261 -0
  57. package/src/config/bundled-skills/media-processing/services/retrieval-service.ts +95 -0
  58. package/src/config/bundled-skills/media-processing/services/timeline-service.ts +267 -0
  59. package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts +301 -0
  60. package/src/config/bundled-skills/media-processing/tools/detect-events.ts +110 -0
  61. package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +190 -0
  62. package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +195 -0
  63. package/src/config/bundled-skills/media-processing/tools/ingest-media.ts +197 -0
  64. package/src/config/bundled-skills/media-processing/tools/media-diagnostics.ts +166 -0
  65. package/src/config/bundled-skills/media-processing/tools/media-status.ts +75 -0
  66. package/src/config/bundled-skills/media-processing/tools/query-media-events.ts +300 -0
  67. package/src/config/bundled-skills/media-processing/tools/recalibrate.ts +235 -0
  68. package/src/config/bundled-skills/media-processing/tools/select-tracking-profile.ts +142 -0
  69. package/src/config/bundled-skills/media-processing/tools/submit-feedback.ts +150 -0
  70. package/src/config/bundled-skills/messaging/SKILL.md +24 -5
  71. package/src/config/bundled-skills/messaging/tools/messaging-send.ts +5 -1
  72. package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
  73. package/src/config/bundled-skills/twitter/SKILL.md +19 -3
  74. package/src/config/defaults.ts +2 -1
  75. package/src/config/schema.ts +9 -3
  76. package/src/config/skills.ts +5 -32
  77. package/src/config/system-prompt.ts +40 -0
  78. package/src/config/templates/IDENTITY.md +2 -2
  79. package/src/config/user-reference.ts +29 -0
  80. package/src/config/vellum-skills/catalog.json +58 -0
  81. package/src/config/vellum-skills/google-oauth-setup/SKILL.md +3 -3
  82. package/src/config/vellum-skills/slack-oauth-setup/SKILL.md +3 -3
  83. package/src/config/vellum-skills/sms-setup/SKILL.md +118 -0
  84. package/src/config/vellum-skills/telegram-setup/SKILL.md +6 -1
  85. package/src/config/vellum-skills/twilio-setup/SKILL.md +76 -6
  86. package/src/daemon/auth-manager.ts +103 -0
  87. package/src/daemon/computer-use-session.ts +8 -1
  88. package/src/daemon/config-watcher.ts +253 -0
  89. package/src/daemon/handlers/config.ts +819 -22
  90. package/src/daemon/handlers/dictation.ts +182 -0
  91. package/src/daemon/handlers/identity.ts +14 -23
  92. package/src/daemon/handlers/index.ts +2 -0
  93. package/src/daemon/handlers/sessions.ts +2 -0
  94. package/src/daemon/handlers/shared.ts +3 -0
  95. package/src/daemon/handlers/skills.ts +6 -7
  96. package/src/daemon/handlers/work-items.ts +15 -7
  97. package/src/daemon/ipc-contract-inventory.json +10 -0
  98. package/src/daemon/ipc-contract.ts +114 -4
  99. package/src/daemon/ipc-handler.ts +87 -0
  100. package/src/daemon/lifecycle.ts +18 -4
  101. package/src/daemon/ride-shotgun-handler.ts +11 -1
  102. package/src/daemon/server.ts +111 -504
  103. package/src/daemon/session-agent-loop.ts +10 -15
  104. package/src/daemon/session-runtime-assembly.ts +115 -44
  105. package/src/daemon/session-tool-setup.ts +2 -0
  106. package/src/daemon/session.ts +19 -2
  107. package/src/inbound/public-ingress-urls.ts +3 -3
  108. package/src/memory/channel-guardian-store.ts +2 -1
  109. package/src/memory/db-connection.ts +28 -0
  110. package/src/memory/db-init.ts +1163 -0
  111. package/src/memory/db.ts +2 -2007
  112. package/src/memory/embedding-backend.ts +79 -11
  113. package/src/memory/indexer.ts +2 -0
  114. package/src/memory/job-handlers/media-processing.ts +100 -0
  115. package/src/memory/job-utils.ts +64 -4
  116. package/src/memory/jobs-store.ts +2 -1
  117. package/src/memory/jobs-worker.ts +11 -1
  118. package/src/memory/media-store.ts +759 -0
  119. package/src/memory/recall-cache.ts +107 -0
  120. package/src/memory/retriever.ts +36 -2
  121. package/src/memory/schema-migration.ts +984 -0
  122. package/src/memory/schema.ts +99 -0
  123. package/src/memory/search/entity.ts +208 -25
  124. package/src/memory/search/ranking.ts +6 -1
  125. package/src/memory/search/types.ts +26 -0
  126. package/src/messaging/provider-types.ts +2 -0
  127. package/src/messaging/providers/sms/adapter.ts +204 -0
  128. package/src/messaging/providers/sms/client.ts +93 -0
  129. package/src/messaging/providers/sms/types.ts +7 -0
  130. package/src/permissions/checker.ts +16 -2
  131. package/src/permissions/prompter.ts +14 -3
  132. package/src/permissions/trust-store.ts +7 -0
  133. package/src/runtime/approval-message-composer.ts +143 -0
  134. package/src/runtime/channel-approvals.ts +29 -7
  135. package/src/runtime/channel-guardian-service.ts +44 -18
  136. package/src/runtime/channel-readiness-service.ts +292 -0
  137. package/src/runtime/channel-readiness-types.ts +29 -0
  138. package/src/runtime/gateway-client.ts +2 -1
  139. package/src/runtime/http-server.ts +65 -28
  140. package/src/runtime/http-types.ts +3 -0
  141. package/src/runtime/routes/call-routes.ts +2 -1
  142. package/src/runtime/routes/channel-routes.ts +237 -103
  143. package/src/runtime/routes/run-routes.ts +7 -1
  144. package/src/runtime/run-orchestrator.ts +43 -3
  145. package/src/security/secret-scanner.ts +218 -0
  146. package/src/skills/frontmatter.ts +63 -0
  147. package/src/skills/slash-commands.ts +23 -0
  148. package/src/skills/vellum-catalog-remote.ts +107 -0
  149. package/src/tools/assets/materialize.ts +2 -2
  150. package/src/tools/browser/auto-navigate.ts +132 -24
  151. package/src/tools/browser/browser-manager.ts +67 -61
  152. package/src/tools/calls/call-start.ts +1 -0
  153. package/src/tools/claude-code/claude-code.ts +55 -3
  154. package/src/tools/credentials/vault.ts +1 -1
  155. package/src/tools/execution-target.ts +11 -1
  156. package/src/tools/executor.ts +10 -2
  157. package/src/tools/network/web-search.ts +1 -1
  158. package/src/tools/skills/vellum-catalog.ts +61 -156
  159. package/src/tools/terminal/parser.ts +21 -5
  160. package/src/tools/types.ts +2 -0
  161. package/src/twitter/router.ts +1 -1
  162. package/src/util/platform.ts +43 -1
  163. package/src/util/retry.ts +4 -4
@@ -0,0 +1,300 @@
1
+ /**
2
+ * Natural language query tool for media events.
3
+ *
4
+ * Accepts a free-text query and optional asset_id, parses the query into
5
+ * structured filters via simple keyword matching, then delegates to the
6
+ * generic retrieval service. Domain-specific keyword mappings live here;
7
+ * the retrieval layer remains fully generic.
8
+ */
9
+
10
+ import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
11
+ import { retrieveEvents, type RetrievalFilters } from '../services/retrieval-service.js';
12
+ import { getTrackingProfile, type CapabilityTier } from '../../../../memory/media-store.js';
13
+ import { getCapabilitiesByTier, getCapabilityByName } from '../services/capability-registry.js';
14
+
15
+ // ---------------------------------------------------------------------------
16
+ // NL query parsing
17
+ // ---------------------------------------------------------------------------
18
+
19
+ /** Maps domain-specific keywords to canonical event type labels. */
20
+ const EVENT_TYPE_KEYWORDS: Record<string, string> = {
21
+ // Basketball
22
+ turnover: 'turnover',
23
+ turnovers: 'turnover',
24
+ steal: 'turnover',
25
+ steals: 'turnover',
26
+ // Soccer / football
27
+ goal: 'goal',
28
+ goals: 'goal',
29
+ score: 'goal',
30
+ scores: 'goal',
31
+ // Generic
32
+ highlight: 'highlight',
33
+ highlights: 'highlight',
34
+ scene_change: 'scene_change',
35
+ 'scene change': 'scene_change',
36
+ 'scene changes': 'scene_change',
37
+ foul: 'foul',
38
+ fouls: 'foul',
39
+ shot: 'shot',
40
+ shots: 'shot',
41
+ rebound: 'rebound',
42
+ rebounds: 'rebound',
43
+ assist: 'assist',
44
+ assists: 'assist',
45
+ block: 'block',
46
+ blocks: 'block',
47
+ transition: 'transition',
48
+ transitions: 'transition',
49
+ fast_break: 'fast_break',
50
+ 'fast break': 'fast_break',
51
+ 'fast breaks': 'fast_break',
52
+ };
53
+
54
+ /**
55
+ * Extract a numeric limit from phrases like "top 5", "first 3", "last 10".
56
+ * Returns undefined if no limit phrase is found.
57
+ */
58
+ function parseLimit(query: string): number | undefined {
59
+ const match = query.match(/(?:top|first|last|show|find|get)\s+(\d+)(?!\s*min)/i);
60
+ if (match) return parseInt(match[1], 10);
61
+ // Also match trailing "N events/moments"
62
+ const trailingMatch = query.match(/(\d+)\s+(?:events?|moments?|plays?|clips?)/i);
63
+ if (trailingMatch) return parseInt(trailingMatch[1], 10);
64
+ return undefined;
65
+ }
66
+
67
+ /**
68
+ * Extract a confidence threshold from the query.
69
+ */
70
+ function parseConfidence(query: string): number | undefined {
71
+ const lower = query.toLowerCase();
72
+ if (lower.includes('high confidence') || lower.includes('very confident') || lower.includes('most confident')) {
73
+ return 0.8;
74
+ }
75
+ if (lower.includes('confident') || lower.includes('medium confidence')) {
76
+ return 0.5;
77
+ }
78
+ // Explicit threshold: "confidence > 0.7" or "confidence above 0.7"
79
+ const explicitMatch = lower.match(/confidence\s*(?:>|above|over|>=)\s*([\d.]+)/);
80
+ if (explicitMatch) return parseFloat(explicitMatch[1]);
81
+ return undefined;
82
+ }
83
+
84
+ /**
85
+ * Extract a time range from the query (returns seconds).
86
+ * Supports phrases like "first half", "second half", "first N minutes",
87
+ * "last N minutes", "between M and N minutes".
88
+ */
89
+ function parseTimeRange(query: string): { startTimeMin?: number; startTimeMax?: number } {
90
+ const lower = query.toLowerCase();
91
+
92
+ // "first half" / "second half" — these are relative and require knowing
93
+ // total duration, which we don't have. Use reasonable game defaults:
94
+ // assume ~48 min game → 2880s, half = 1440s
95
+ if (lower.includes('first half')) {
96
+ return { startTimeMin: 0, startTimeMax: 1440 };
97
+ }
98
+ if (lower.includes('second half')) {
99
+ return { startTimeMin: 1440 };
100
+ }
101
+
102
+ // "first N minutes"
103
+ const firstNMin = lower.match(/first\s+(\d+)\s*min/);
104
+ if (firstNMin) {
105
+ return { startTimeMin: 0, startTimeMax: parseInt(firstNMin[1], 10) * 60 };
106
+ }
107
+
108
+ // "last N minutes"
109
+ // Without knowing total duration we can't compute this precisely,
110
+ // so we skip it and let the retrieval return all results.
111
+
112
+ // "between M and N minutes"
113
+ const betweenMatch = lower.match(/between\s+(\d+)\s*(?:and|to|-)\s*(\d+)\s*min/);
114
+ if (betweenMatch) {
115
+ return {
116
+ startTimeMin: parseInt(betweenMatch[1], 10) * 60,
117
+ startTimeMax: parseInt(betweenMatch[2], 10) * 60,
118
+ };
119
+ }
120
+
121
+ return {};
122
+ }
123
+
124
+ /**
125
+ * Extract the event type from the query by matching known keywords.
126
+ */
127
+ function parseEventType(query: string): string | undefined {
128
+ const lower = query.toLowerCase();
129
+
130
+ // Check multi-word keywords first (longer matches win)
131
+ const multiWordKeys = Object.keys(EVENT_TYPE_KEYWORDS)
132
+ .filter((k) => k.includes(' ') || k.includes('_'))
133
+ .sort((a, b) => b.length - a.length);
134
+
135
+ for (const key of multiWordKeys) {
136
+ if (lower.includes(key)) return EVENT_TYPE_KEYWORDS[key];
137
+ }
138
+
139
+ // Check single-word keywords
140
+ const words = lower.replace(/[^\w\s]/g, '').split(/\s+/);
141
+ for (const word of words) {
142
+ if (EVENT_TYPE_KEYWORDS[word]) return EVENT_TYPE_KEYWORDS[word];
143
+ }
144
+
145
+ return undefined;
146
+ }
147
+
148
+ /**
149
+ * Parse a natural language query into structured retrieval filters.
150
+ */
151
+ function parseQuery(query: string, assetId?: string): RetrievalFilters {
152
+ const eventType = parseEventType(query);
153
+ const limit = parseLimit(query);
154
+ const minConfidence = parseConfidence(query);
155
+ const timeRange = parseTimeRange(query);
156
+
157
+ return {
158
+ assetId,
159
+ eventType,
160
+ minConfidence,
161
+ limit: limit ?? 10,
162
+ sortBy: 'confidence',
163
+ ...timeRange,
164
+ };
165
+ }
166
+
167
+ // ---------------------------------------------------------------------------
168
+ // Formatting
169
+ // ---------------------------------------------------------------------------
170
+
171
+ function formatTimestamp(seconds: number): string {
172
+ const mins = Math.floor(seconds / 60);
173
+ const secs = Math.floor(seconds % 60);
174
+ return `${mins}:${secs.toString().padStart(2, '0')}`;
175
+ }
176
+
177
+ // ---------------------------------------------------------------------------
178
+ // Tool entry point
179
+ // ---------------------------------------------------------------------------
180
+
181
+ export async function run(
182
+ input: Record<string, unknown>,
183
+ _context: ToolContext,
184
+ ): Promise<ToolExecutionResult> {
185
+ const query = input.query as string | undefined;
186
+ if (!query) {
187
+ return { content: 'query is required.', isError: true };
188
+ }
189
+
190
+ const assetId = input.asset_id as string | undefined;
191
+ if (!assetId) {
192
+ return { content: 'asset_id is required to scope the query to a specific media asset.', isError: true };
193
+ }
194
+
195
+ const filters = parseQuery(query, assetId);
196
+
197
+ // Retrieve without limit so we can apply capability filtering first, then limit
198
+ const userLimit = filters.limit;
199
+ const result = retrieveEvents({ ...filters, limit: 0 });
200
+
201
+ // Determine which capabilities are allowed based on the tracking profile
202
+ const profile = getTrackingProfile(assetId);
203
+
204
+ let allowedEventTypes: Set<string> | null = null;
205
+ const tierForEventType = new Map<string, CapabilityTier>();
206
+
207
+ if (profile) {
208
+ // Use the explicit profile: only enabled capabilities pass
209
+ allowedEventTypes = new Set<string>();
210
+ for (const [capName, entry] of Object.entries(profile.capabilities)) {
211
+ if (entry.enabled) {
212
+ allowedEventTypes.add(capName);
213
+ tierForEventType.set(capName, entry.tier);
214
+ }
215
+ }
216
+ } else {
217
+ // No profile: default to 'ready' tier capabilities only
218
+ const readyCaps = getCapabilitiesByTier('ready');
219
+ if (readyCaps.length > 0) {
220
+ allowedEventTypes = new Set(readyCaps.map((c) => c.name));
221
+ for (const cap of readyCaps) {
222
+ tierForEventType.set(cap.name, 'ready');
223
+ }
224
+ }
225
+ // If no capabilities are registered at all, allow everything (pass null)
226
+ }
227
+
228
+ // Filter events by allowed capabilities, then apply the user-requested limit
229
+ let filteredEvents = result.events;
230
+ if (allowedEventTypes !== null) {
231
+ filteredEvents = filteredEvents.filter((e) => allowedEventTypes!.has(e.eventType));
232
+ }
233
+ if (userLimit && filteredEvents.length > userLimit) {
234
+ filteredEvents = filteredEvents.slice(0, userLimit);
235
+ }
236
+
237
+ if (filteredEvents.length === 0) {
238
+ const parts = ['No matching events found'];
239
+ if (filters.eventType) parts.push(`for event type "${filters.eventType}"`);
240
+ if (filters.minConfidence) parts.push(`with confidence >= ${filters.minConfidence}`);
241
+ if (!profile) parts.push('(defaulting to ready-tier capabilities only)');
242
+ parts.push(`in asset ${assetId}.`);
243
+ return { content: parts.join(' '), isError: false };
244
+ }
245
+
246
+ const tierLabels: Record<string, string> = {
247
+ ready: '[Ready]',
248
+ beta: '[Beta]',
249
+ experimental: '[Experimental]',
250
+ };
251
+
252
+ const tierDisclaimers: Record<string, string> = {
253
+ beta: 'Beta results may have accuracy gaps.',
254
+ experimental: 'Experimental results are early-stage; expect noise.',
255
+ };
256
+
257
+ const eventSummaries = filteredEvents.map((e, i) => {
258
+ const tier = tierForEventType.get(e.eventType) ?? getCapabilityByName(e.eventType)?.tier;
259
+ const tierLabel = tier ? tierLabels[tier] ?? `[${tier}]` : '[Ready]';
260
+ const disclaimer = tier ? tierDisclaimers[tier] : undefined;
261
+
262
+ return {
263
+ rank: i + 1,
264
+ id: e.id,
265
+ eventType: e.eventType,
266
+ tierLabel,
267
+ ...(disclaimer ? { confidenceDisclaimer: disclaimer } : {}),
268
+ timeRange: `${formatTimestamp(e.startTime)} – ${formatTimestamp(e.endTime)}`,
269
+ startTime: e.startTime,
270
+ endTime: e.endTime,
271
+ confidence: Math.round(e.confidence * 100) / 100,
272
+ reasons: e.reasons,
273
+ metadata: e.metadata,
274
+ };
275
+ });
276
+
277
+ // Collect disclaimers for any non-ready tiers present in results
278
+ const activeTiers = new Set(eventSummaries.map((e) => e.tierLabel));
279
+ const disclaimers: string[] = [];
280
+ if (activeTiers.has('[Beta]')) disclaimers.push(tierDisclaimers.beta);
281
+ if (activeTiers.has('[Experimental]')) disclaimers.push(tierDisclaimers.experimental);
282
+
283
+ return {
284
+ content: JSON.stringify({
285
+ query,
286
+ parsedFilters: {
287
+ eventType: filters.eventType ?? null,
288
+ minConfidence: filters.minConfidence ?? null,
289
+ limit: filters.limit,
290
+ startTimeMin: filters.startTimeMin ?? null,
291
+ startTimeMax: filters.startTimeMax ?? null,
292
+ },
293
+ trackingProfile: profile ? 'custom' : 'default (ready tier only)',
294
+ ...(disclaimers.length > 0 ? { disclaimers } : {}),
295
+ totalResults: eventSummaries.length,
296
+ events: eventSummaries,
297
+ }, null, 2),
298
+ isError: false,
299
+ };
300
+ }
@@ -0,0 +1,235 @@
1
+ /**
2
+ * Recalibration tool for media event detection.
3
+ *
4
+ * Reads all feedback for an asset, analyzes correction patterns, and
5
+ * re-ranks existing events using updated heuristics. Updates confidence
6
+ * scores in the media_events table.
7
+ *
8
+ * All interfaces are generic — works for any event type.
9
+ */
10
+
11
+ import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
12
+ import { getFeedbackForAsset, type EventFeedback } from '../services/feedback-store.js';
13
+ import { aggregateFeedback } from '../services/feedback-aggregation.js';
14
+ import { getEventsForAsset, getMediaAssetById } from '../../../../memory/media-store.js';
15
+ import { getDb } from '../../../../memory/db.js';
16
+ import { mediaEvents } from '../../../../memory/schema.js';
17
+ import { eq } from 'drizzle-orm';
18
+
19
+ // ---------------------------------------------------------------------------
20
+ // Types
21
+ // ---------------------------------------------------------------------------
22
+
23
+ interface RecalibrationAdjustment {
24
+ eventType: string;
25
+ action: string;
26
+ detail: string;
27
+ }
28
+
29
+ interface EventUpdate {
30
+ eventId: string;
31
+ eventType: string;
32
+ oldConfidence: number;
33
+ newConfidence: number;
34
+ }
35
+
36
+ // ---------------------------------------------------------------------------
37
+ // Tool entry point
38
+ // ---------------------------------------------------------------------------
39
+
40
+ export async function run(
41
+ input: Record<string, unknown>,
42
+ context: ToolContext,
43
+ ): Promise<ToolExecutionResult> {
44
+ const assetId = input.asset_id as string | undefined;
45
+ if (!assetId) {
46
+ return { content: 'asset_id is required.', isError: true };
47
+ }
48
+
49
+ const asset = getMediaAssetById(assetId);
50
+ if (!asset) {
51
+ return { content: `Asset "${assetId}" not found.`, isError: true };
52
+ }
53
+
54
+ const allFeedback = getFeedbackForAsset(assetId);
55
+ if (allFeedback.length === 0) {
56
+ return {
57
+ content: JSON.stringify({
58
+ message: 'No feedback found for this asset. Submit feedback first, then recalibrate.',
59
+ assetId,
60
+ }, null, 2),
61
+ isError: false,
62
+ };
63
+ }
64
+
65
+ const aggregation = aggregateFeedback(assetId);
66
+ const allEvents = getEventsForAsset(assetId);
67
+
68
+ // Build a map of event ID to its feedback entries
69
+ const feedbackByEventId = new Map<string, EventFeedback[]>();
70
+ for (const fb of allFeedback) {
71
+ if (!feedbackByEventId.has(fb.eventId)) {
72
+ feedbackByEventId.set(fb.eventId, []);
73
+ }
74
+ feedbackByEventId.get(fb.eventId)!.push(fb);
75
+ }
76
+
77
+ // Build a map of event ID to event type for filtering feedback by type
78
+ const eventTypeById = new Map<string, string>();
79
+ for (const ev of allEvents) {
80
+ eventTypeById.set(ev.id, ev.eventType);
81
+ }
82
+
83
+ const adjustments: RecalibrationAdjustment[] = [];
84
+ const eventUpdates: EventUpdate[] = [];
85
+ const db = getDb();
86
+
87
+ // Analyze patterns per event type
88
+ for (const stats of aggregation.statsByEventType) {
89
+ const { eventType, correct, incorrect, boundaryEdit, missed } = stats;
90
+ const totalReviewed = correct + incorrect + boundaryEdit + missed;
91
+ if (totalReviewed === 0) continue;
92
+
93
+ // Pattern 1: High false positive rate — penalize low-confidence events
94
+ const falsePositiveRate = totalReviewed > 0 ? incorrect / totalReviewed : 0;
95
+ if (falsePositiveRate > 0.3 && incorrect >= 2) {
96
+ adjustments.push({
97
+ eventType,
98
+ action: 'penalize_low_confidence',
99
+ detail: `False positive rate ${(falsePositiveRate * 100).toFixed(1)}% (${incorrect}/${totalReviewed}) — reducing confidence on unreviewed events of this type`,
100
+ });
101
+ }
102
+
103
+ // Pattern 2: Many missed events — note for threshold adjustment
104
+ if (missed >= 2) {
105
+ adjustments.push({
106
+ eventType,
107
+ action: 'note_missed_events',
108
+ detail: `${missed} missed events reported — consider lowering detection threshold or adding detection rules for this type`,
109
+ });
110
+ }
111
+
112
+ // Pattern 3: Boundary edits — compute average adjustment
113
+ if (boundaryEdit >= 1) {
114
+ const boundaryFeedback = allFeedback.filter(
115
+ (fb) => fb.feedbackType === 'boundary_edit' && eventTypeById.get(fb.eventId) === eventType,
116
+ );
117
+ let startAdjTotal = 0;
118
+ let endAdjTotal = 0;
119
+ let startAdjCount = 0;
120
+ let endAdjCount = 0;
121
+
122
+ for (const fb of boundaryFeedback) {
123
+ if (fb.originalStartTime !== null && fb.correctedStartTime !== null) {
124
+ startAdjTotal += fb.correctedStartTime - fb.originalStartTime;
125
+ startAdjCount++;
126
+ }
127
+ if (fb.originalEndTime !== null && fb.correctedEndTime !== null) {
128
+ endAdjTotal += fb.correctedEndTime - fb.originalEndTime;
129
+ endAdjCount++;
130
+ }
131
+ }
132
+
133
+ const avgStartAdj = startAdjCount > 0 ? startAdjTotal / startAdjCount : 0;
134
+ const avgEndAdj = endAdjCount > 0 ? endAdjTotal / endAdjCount : 0;
135
+
136
+ if (startAdjCount > 0 || endAdjCount > 0) {
137
+ adjustments.push({
138
+ eventType,
139
+ action: 'boundary_correction_pattern',
140
+ detail: `Average boundary adjustment: start ${avgStartAdj >= 0 ? '+' : ''}${avgStartAdj.toFixed(2)}s (n=${startAdjCount}), end ${avgEndAdj >= 0 ? '+' : ''}${avgEndAdj.toFixed(2)}s (n=${endAdjCount})`,
141
+ });
142
+ }
143
+ }
144
+ }
145
+
146
+ // Re-rank events: adjust confidence based on feedback
147
+ for (const event of allEvents) {
148
+ const eventFeedback = feedbackByEventId.get(event.id);
149
+ if (!eventFeedback || eventFeedback.length === 0) {
150
+ // For events without direct feedback, apply type-level adjustments
151
+ const stats = aggregation.statsByEventType.find((s) => s.eventType === event.eventType);
152
+ if (stats) {
153
+ const totalReviewed = stats.correct + stats.incorrect + stats.boundaryEdit + stats.missed;
154
+ const falsePositiveRate = totalReviewed > 0 ? stats.incorrect / totalReviewed : 0;
155
+
156
+ // If high false positive rate for this type, reduce unreviewed event confidence
157
+ if (falsePositiveRate > 0.3 && totalReviewed >= 3) {
158
+ const penalty = Math.min(falsePositiveRate * 0.3, 0.2);
159
+ const newConfidence = Math.max(0.05, event.confidence - penalty);
160
+ if (newConfidence !== event.confidence) {
161
+ db.update(mediaEvents)
162
+ .set({ confidence: newConfidence })
163
+ .where(eq(mediaEvents.id, event.id))
164
+ .run();
165
+ eventUpdates.push({
166
+ eventId: event.id,
167
+ eventType: event.eventType,
168
+ oldConfidence: event.confidence,
169
+ newConfidence,
170
+ });
171
+ }
172
+ }
173
+ }
174
+ continue;
175
+ }
176
+
177
+ // Direct feedback: adjust confidence based on the latest feedback
178
+ const latestFeedback = eventFeedback.sort((a, b) => b.createdAt - a.createdAt)[0];
179
+ let newConfidence = event.confidence;
180
+
181
+ switch (latestFeedback.feedbackType) {
182
+ case 'correct':
183
+ // Boost confidence toward 1.0
184
+ newConfidence = Math.min(1.0, event.confidence + (1.0 - event.confidence) * 0.3);
185
+ break;
186
+ case 'incorrect':
187
+ // Sharply reduce confidence
188
+ newConfidence = Math.max(0.05, event.confidence * 0.3);
189
+ break;
190
+ case 'boundary_edit':
191
+ // Slight confidence boost (event was real but boundaries were off)
192
+ newConfidence = Math.min(1.0, event.confidence + (1.0 - event.confidence) * 0.15);
193
+ break;
194
+ case 'missed':
195
+ // User-reported events keep their initial confidence
196
+ break;
197
+ }
198
+
199
+ newConfidence = Math.round(newConfidence * 1000) / 1000;
200
+
201
+ if (newConfidence !== event.confidence) {
202
+ db.update(mediaEvents)
203
+ .set({ confidence: newConfidence })
204
+ .where(eq(mediaEvents.id, event.id))
205
+ .run();
206
+ eventUpdates.push({
207
+ eventId: event.id,
208
+ eventType: event.eventType,
209
+ oldConfidence: event.confidence,
210
+ newConfidence,
211
+ });
212
+ }
213
+ }
214
+
215
+ context.onOutput?.(`Recalibrated ${eventUpdates.length} events based on ${allFeedback.length} feedback entries.\n`);
216
+
217
+ return {
218
+ content: JSON.stringify({
219
+ message: `Recalibration complete for asset ${assetId}`,
220
+ assetId,
221
+ totalFeedbackEntries: allFeedback.length,
222
+ adjustments,
223
+ eventsUpdated: eventUpdates.length,
224
+ eventUpdates: eventUpdates.map((u) => ({
225
+ eventId: u.eventId,
226
+ eventType: u.eventType,
227
+ oldConfidence: u.oldConfidence,
228
+ newConfidence: u.newConfidence,
229
+ delta: Math.round((u.newConfidence - u.oldConfidence) * 1000) / 1000,
230
+ })),
231
+ aggregation: aggregation.statsByEventType,
232
+ }, null, 2),
233
+ isError: false,
234
+ };
235
+ }
@@ -0,0 +1,142 @@
1
+ /**
2
+ * Select and persist a tracking profile for a media asset.
3
+ *
4
+ * When called without capabilities, returns the available capabilities
5
+ * organized by tier so the user can choose. When called with capabilities,
6
+ * validates them against the registry and stores the profile.
7
+ */
8
+
9
+ import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
10
+ import { getMediaAssetById, setTrackingProfile, getTrackingProfile, type CapabilityProfile } from '../../../../memory/media-store.js';
11
+ import {
12
+ getCapabilities,
13
+ getCapabilityByName,
14
+ getRegisteredDomains,
15
+ type Capability,
16
+ } from '../services/capability-registry.js';
17
+
18
+ // ---------------------------------------------------------------------------
19
+ // Helpers
20
+ // ---------------------------------------------------------------------------
21
+
22
+ function groupByTier(caps: Capability[]): Record<string, Capability[]> {
23
+ const groups: Record<string, Capability[]> = { ready: [], beta: [], experimental: [] };
24
+ for (const cap of caps) {
25
+ if (!groups[cap.tier]) groups[cap.tier] = [];
26
+ groups[cap.tier].push(cap);
27
+ }
28
+ return groups;
29
+ }
30
+
31
+ // ---------------------------------------------------------------------------
32
+ // Tool entry point
33
+ // ---------------------------------------------------------------------------
34
+
35
+ export async function run(
36
+ input: Record<string, unknown>,
37
+ _context: ToolContext,
38
+ ): Promise<ToolExecutionResult> {
39
+ const assetId = input.asset_id as string | undefined;
40
+ if (!assetId) {
41
+ return { content: 'asset_id is required.', isError: true };
42
+ }
43
+
44
+ const asset = getMediaAssetById(assetId);
45
+ if (!asset) {
46
+ return { content: `No media asset found with id "${assetId}".`, isError: true };
47
+ }
48
+
49
+ const requestedCapabilities = input.capabilities as string[] | undefined;
50
+
51
+ // If no capabilities provided, return available options for the user to choose
52
+ if (!requestedCapabilities || requestedCapabilities.length === 0) {
53
+ const allCaps = getCapabilities();
54
+ const domains = getRegisteredDomains();
55
+ const byTier = groupByTier(allCaps);
56
+
57
+ const currentProfile = getTrackingProfile(assetId);
58
+
59
+ return {
60
+ content: JSON.stringify({
61
+ message: 'No capabilities specified. Here are the available capabilities organized by tier. Call again with a capabilities array to enable specific ones.',
62
+ domains,
63
+ availableCapabilities: {
64
+ ready: byTier.ready.map((c) => ({
65
+ name: c.name,
66
+ description: c.description,
67
+ domain: c.domain,
68
+ granularity: c.granularity ?? null,
69
+ })),
70
+ beta: byTier.beta.map((c) => ({
71
+ name: c.name,
72
+ description: c.description,
73
+ domain: c.domain,
74
+ granularity: c.granularity ?? null,
75
+ note: 'Beta: functional but may have accuracy gaps',
76
+ })),
77
+ experimental: byTier.experimental.map((c) => ({
78
+ name: c.name,
79
+ description: c.description,
80
+ domain: c.domain,
81
+ granularity: c.granularity ?? null,
82
+ note: 'Experimental: early-stage, expect noise in results',
83
+ })),
84
+ },
85
+ currentProfile: currentProfile ? currentProfile.capabilities : null,
86
+ }, null, 2),
87
+ isError: false,
88
+ };
89
+ }
90
+
91
+ // Validate requested capabilities exist in the registry
92
+ const capabilities: CapabilityProfile = {};
93
+ const unknownCaps: string[] = [];
94
+
95
+ for (const capName of requestedCapabilities) {
96
+ const registered = getCapabilityByName(capName);
97
+ if (!registered) {
98
+ unknownCaps.push(capName);
99
+ continue;
100
+ }
101
+ capabilities[capName] = { enabled: true, tier: registered.tier };
102
+ }
103
+
104
+ if (unknownCaps.length > 0) {
105
+ const allCaps = getCapabilities();
106
+ return {
107
+ content: JSON.stringify({
108
+ error: `Unknown capabilities: ${unknownCaps.join(', ')}`,
109
+ availableCapabilities: allCaps.map((c) => c.name),
110
+ }, null, 2),
111
+ isError: true,
112
+ };
113
+ }
114
+
115
+ // Store the profile
116
+ const profile = setTrackingProfile(assetId, capabilities);
117
+
118
+ // Build response with tier labels
119
+ const profileSummary: Record<string, { enabled: boolean; tier: string; tierLabel: string }> = {};
120
+ for (const [name, entry] of Object.entries(profile.capabilities)) {
121
+ const tierLabels: Record<string, string> = {
122
+ ready: '[Ready]',
123
+ beta: '[Beta]',
124
+ experimental: '[Experimental]',
125
+ };
126
+ profileSummary[name] = {
127
+ enabled: entry.enabled,
128
+ tier: entry.tier,
129
+ tierLabel: tierLabels[entry.tier] ?? `[${entry.tier}]`,
130
+ };
131
+ }
132
+
133
+ return {
134
+ content: JSON.stringify({
135
+ message: 'Tracking profile saved.',
136
+ assetId: profile.assetId,
137
+ profileId: profile.id,
138
+ capabilities: profileSummary,
139
+ }, null, 2),
140
+ isError: false,
141
+ };
142
+ }