@vellumai/assistant 0.3.4 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +2 -0
- package/README.md +37 -2
- package/package.json +1 -1
- package/scripts/ipc/generate-swift.ts +13 -0
- package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +100 -0
- package/src/__tests__/approval-hardcoded-copy-guard.test.ts +41 -0
- package/src/__tests__/approval-message-composer.test.ts +253 -0
- package/src/__tests__/call-domain.test.ts +12 -2
- package/src/__tests__/call-orchestrator.test.ts +70 -1
- package/src/__tests__/call-routes-http.test.ts +27 -2
- package/src/__tests__/channel-approval-routes.test.ts +21 -17
- package/src/__tests__/channel-approvals.test.ts +48 -1
- package/src/__tests__/channel-guardian.test.ts +74 -22
- package/src/__tests__/channel-readiness-service.test.ts +257 -0
- package/src/__tests__/config-schema.test.ts +2 -1
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/daemon-lifecycle.test.ts +13 -12
- package/src/__tests__/dictation-mode-detection.test.ts +63 -0
- package/src/__tests__/entity-search.test.ts +615 -0
- package/src/__tests__/handlers-twilio-config.test.ts +407 -0
- package/src/__tests__/ipc-snapshot.test.ts +63 -0
- package/src/__tests__/messaging-send-tool.test.ts +65 -0
- package/src/__tests__/run-orchestrator-assistant-events.test.ts +4 -0
- package/src/__tests__/run-orchestrator.test.ts +22 -0
- package/src/__tests__/session-runtime-assembly.test.ts +85 -1
- package/src/__tests__/sms-messaging-provider.test.ts +125 -0
- package/src/__tests__/twilio-routes.test.ts +39 -3
- package/src/__tests__/twitter-cli-error-shaping.test.ts +2 -2
- package/src/__tests__/web-search.test.ts +1 -1
- package/src/__tests__/work-item-output.test.ts +110 -0
- package/src/calls/call-domain.ts +8 -5
- package/src/calls/call-orchestrator.ts +22 -11
- package/src/calls/twilio-config.ts +17 -11
- package/src/calls/twilio-rest.ts +276 -0
- package/src/calls/twilio-routes.ts +39 -1
- package/src/config/bundled-skills/knowledge-graph/SKILL.md +15 -0
- package/src/config/bundled-skills/knowledge-graph/TOOLS.json +56 -0
- package/src/config/bundled-skills/knowledge-graph/tools/graph-query.ts +185 -0
- package/src/config/bundled-skills/media-processing/SKILL.md +199 -0
- package/src/config/bundled-skills/media-processing/TOOLS.json +320 -0
- package/src/config/bundled-skills/media-processing/services/capability-registry.ts +137 -0
- package/src/config/bundled-skills/media-processing/services/event-detection-service.ts +280 -0
- package/src/config/bundled-skills/media-processing/services/feedback-aggregation.ts +144 -0
- package/src/config/bundled-skills/media-processing/services/feedback-store.ts +136 -0
- package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +261 -0
- package/src/config/bundled-skills/media-processing/services/retrieval-service.ts +95 -0
- package/src/config/bundled-skills/media-processing/services/timeline-service.ts +267 -0
- package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts +301 -0
- package/src/config/bundled-skills/media-processing/tools/detect-events.ts +110 -0
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +190 -0
- package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +195 -0
- package/src/config/bundled-skills/media-processing/tools/ingest-media.ts +197 -0
- package/src/config/bundled-skills/media-processing/tools/media-diagnostics.ts +166 -0
- package/src/config/bundled-skills/media-processing/tools/media-status.ts +75 -0
- package/src/config/bundled-skills/media-processing/tools/query-media-events.ts +300 -0
- package/src/config/bundled-skills/media-processing/tools/recalibrate.ts +235 -0
- package/src/config/bundled-skills/media-processing/tools/select-tracking-profile.ts +142 -0
- package/src/config/bundled-skills/media-processing/tools/submit-feedback.ts +150 -0
- package/src/config/bundled-skills/messaging/SKILL.md +21 -6
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +5 -1
- package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
- package/src/config/bundled-skills/twitter/SKILL.md +19 -3
- package/src/config/defaults.ts +2 -1
- package/src/config/schema.ts +9 -3
- package/src/config/system-prompt.ts +24 -0
- package/src/config/templates/IDENTITY.md +2 -2
- package/src/config/vellum-skills/catalog.json +6 -0
- package/src/config/vellum-skills/google-oauth-setup/SKILL.md +3 -3
- package/src/config/vellum-skills/slack-oauth-setup/SKILL.md +3 -3
- package/src/config/vellum-skills/sms-setup/SKILL.md +118 -0
- package/src/config/vellum-skills/twilio-setup/SKILL.md +40 -8
- package/src/daemon/handlers/config.ts +783 -9
- package/src/daemon/handlers/dictation.ts +182 -0
- package/src/daemon/handlers/identity.ts +14 -23
- package/src/daemon/handlers/index.ts +2 -0
- package/src/daemon/handlers/sessions.ts +2 -0
- package/src/daemon/handlers/shared.ts +3 -0
- package/src/daemon/handlers/work-items.ts +15 -7
- package/src/daemon/ipc-contract-inventory.json +10 -0
- package/src/daemon/ipc-contract.ts +108 -4
- package/src/daemon/lifecycle.ts +2 -0
- package/src/daemon/ride-shotgun-handler.ts +1 -1
- package/src/daemon/server.ts +6 -2
- package/src/daemon/session-agent-loop.ts +5 -1
- package/src/daemon/session-runtime-assembly.ts +55 -0
- package/src/daemon/session-tool-setup.ts +2 -0
- package/src/daemon/session.ts +11 -1
- package/src/inbound/public-ingress-urls.ts +3 -3
- package/src/memory/channel-guardian-store.ts +2 -1
- package/src/memory/db-init.ts +144 -0
- package/src/memory/job-handlers/media-processing.ts +100 -0
- package/src/memory/jobs-store.ts +2 -1
- package/src/memory/jobs-worker.ts +4 -0
- package/src/memory/media-store.ts +759 -0
- package/src/memory/retriever.ts +6 -1
- package/src/memory/schema.ts +98 -0
- package/src/memory/search/entity.ts +208 -25
- package/src/memory/search/ranking.ts +6 -1
- package/src/memory/search/types.ts +24 -0
- package/src/messaging/provider-types.ts +2 -0
- package/src/messaging/providers/sms/adapter.ts +204 -0
- package/src/messaging/providers/sms/client.ts +93 -0
- package/src/messaging/providers/sms/types.ts +7 -0
- package/src/permissions/checker.ts +16 -2
- package/src/runtime/approval-message-composer.ts +143 -0
- package/src/runtime/channel-approvals.ts +12 -4
- package/src/runtime/channel-guardian-service.ts +44 -18
- package/src/runtime/channel-readiness-service.ts +292 -0
- package/src/runtime/channel-readiness-types.ts +29 -0
- package/src/runtime/http-server.ts +53 -27
- package/src/runtime/http-types.ts +3 -0
- package/src/runtime/routes/call-routes.ts +2 -1
- package/src/runtime/routes/channel-routes.ts +67 -21
- package/src/runtime/run-orchestrator.ts +35 -2
- package/src/tools/assets/materialize.ts +2 -2
- package/src/tools/calls/call-start.ts +1 -0
- package/src/tools/credentials/vault.ts +1 -1
- package/src/tools/execution-target.ts +11 -1
- package/src/tools/network/web-search.ts +1 -1
- package/src/tools/types.ts +2 -0
- package/src/twitter/router.ts +1 -1
- package/src/util/platform.ts +35 -0
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Natural language query tool for media events.
|
|
3
|
+
*
|
|
4
|
+
* Accepts a free-text query and optional asset_id, parses the query into
|
|
5
|
+
* structured filters via simple keyword matching, then delegates to the
|
|
6
|
+
* generic retrieval service. Domain-specific keyword mappings live here;
|
|
7
|
+
* the retrieval layer remains fully generic.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
|
|
11
|
+
import { retrieveEvents, type RetrievalFilters } from '../services/retrieval-service.js';
|
|
12
|
+
import { getTrackingProfile, type CapabilityTier } from '../../../../memory/media-store.js';
|
|
13
|
+
import { getCapabilitiesByTier, getCapabilityByName } from '../services/capability-registry.js';
|
|
14
|
+
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
// NL query parsing
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
|
|
19
|
+
/** Maps domain-specific keywords to canonical event type labels. */
|
|
20
|
+
const EVENT_TYPE_KEYWORDS: Record<string, string> = {
|
|
21
|
+
// Basketball
|
|
22
|
+
turnover: 'turnover',
|
|
23
|
+
turnovers: 'turnover',
|
|
24
|
+
steal: 'turnover',
|
|
25
|
+
steals: 'turnover',
|
|
26
|
+
// Soccer / football
|
|
27
|
+
goal: 'goal',
|
|
28
|
+
goals: 'goal',
|
|
29
|
+
score: 'goal',
|
|
30
|
+
scores: 'goal',
|
|
31
|
+
// Generic
|
|
32
|
+
highlight: 'highlight',
|
|
33
|
+
highlights: 'highlight',
|
|
34
|
+
scene_change: 'scene_change',
|
|
35
|
+
'scene change': 'scene_change',
|
|
36
|
+
'scene changes': 'scene_change',
|
|
37
|
+
foul: 'foul',
|
|
38
|
+
fouls: 'foul',
|
|
39
|
+
shot: 'shot',
|
|
40
|
+
shots: 'shot',
|
|
41
|
+
rebound: 'rebound',
|
|
42
|
+
rebounds: 'rebound',
|
|
43
|
+
assist: 'assist',
|
|
44
|
+
assists: 'assist',
|
|
45
|
+
block: 'block',
|
|
46
|
+
blocks: 'block',
|
|
47
|
+
transition: 'transition',
|
|
48
|
+
transitions: 'transition',
|
|
49
|
+
fast_break: 'fast_break',
|
|
50
|
+
'fast break': 'fast_break',
|
|
51
|
+
'fast breaks': 'fast_break',
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Extract a numeric limit from phrases like "top 5", "first 3", "last 10".
|
|
56
|
+
* Returns undefined if no limit phrase is found.
|
|
57
|
+
*/
|
|
58
|
+
function parseLimit(query: string): number | undefined {
|
|
59
|
+
const match = query.match(/(?:top|first|last|show|find|get)\s+(\d+)(?!\s*min)/i);
|
|
60
|
+
if (match) return parseInt(match[1], 10);
|
|
61
|
+
// Also match trailing "N events/moments"
|
|
62
|
+
const trailingMatch = query.match(/(\d+)\s+(?:events?|moments?|plays?|clips?)/i);
|
|
63
|
+
if (trailingMatch) return parseInt(trailingMatch[1], 10);
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Extract a confidence threshold from the query.
|
|
69
|
+
*/
|
|
70
|
+
function parseConfidence(query: string): number | undefined {
|
|
71
|
+
const lower = query.toLowerCase();
|
|
72
|
+
if (lower.includes('high confidence') || lower.includes('very confident') || lower.includes('most confident')) {
|
|
73
|
+
return 0.8;
|
|
74
|
+
}
|
|
75
|
+
if (lower.includes('confident') || lower.includes('medium confidence')) {
|
|
76
|
+
return 0.5;
|
|
77
|
+
}
|
|
78
|
+
// Explicit threshold: "confidence > 0.7" or "confidence above 0.7"
|
|
79
|
+
const explicitMatch = lower.match(/confidence\s*(?:>|above|over|>=)\s*([\d.]+)/);
|
|
80
|
+
if (explicitMatch) return parseFloat(explicitMatch[1]);
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Extract a time range from the query (returns seconds).
|
|
86
|
+
* Supports phrases like "first half", "second half", "first N minutes",
|
|
87
|
+
* "last N minutes", "between M and N minutes".
|
|
88
|
+
*/
|
|
89
|
+
function parseTimeRange(query: string): { startTimeMin?: number; startTimeMax?: number } {
|
|
90
|
+
const lower = query.toLowerCase();
|
|
91
|
+
|
|
92
|
+
// "first half" / "second half" — these are relative and require knowing
|
|
93
|
+
// total duration, which we don't have. Use reasonable game defaults:
|
|
94
|
+
// assume ~48 min game → 2880s, half = 1440s
|
|
95
|
+
if (lower.includes('first half')) {
|
|
96
|
+
return { startTimeMin: 0, startTimeMax: 1440 };
|
|
97
|
+
}
|
|
98
|
+
if (lower.includes('second half')) {
|
|
99
|
+
return { startTimeMin: 1440 };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// "first N minutes"
|
|
103
|
+
const firstNMin = lower.match(/first\s+(\d+)\s*min/);
|
|
104
|
+
if (firstNMin) {
|
|
105
|
+
return { startTimeMin: 0, startTimeMax: parseInt(firstNMin[1], 10) * 60 };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// "last N minutes"
|
|
109
|
+
// Without knowing total duration we can't compute this precisely,
|
|
110
|
+
// so we skip it and let the retrieval return all results.
|
|
111
|
+
|
|
112
|
+
// "between M and N minutes"
|
|
113
|
+
const betweenMatch = lower.match(/between\s+(\d+)\s*(?:and|to|-)\s*(\d+)\s*min/);
|
|
114
|
+
if (betweenMatch) {
|
|
115
|
+
return {
|
|
116
|
+
startTimeMin: parseInt(betweenMatch[1], 10) * 60,
|
|
117
|
+
startTimeMax: parseInt(betweenMatch[2], 10) * 60,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
return {};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Extract the event type from the query by matching known keywords.
|
|
126
|
+
*/
|
|
127
|
+
function parseEventType(query: string): string | undefined {
|
|
128
|
+
const lower = query.toLowerCase();
|
|
129
|
+
|
|
130
|
+
// Check multi-word keywords first (longer matches win)
|
|
131
|
+
const multiWordKeys = Object.keys(EVENT_TYPE_KEYWORDS)
|
|
132
|
+
.filter((k) => k.includes(' ') || k.includes('_'))
|
|
133
|
+
.sort((a, b) => b.length - a.length);
|
|
134
|
+
|
|
135
|
+
for (const key of multiWordKeys) {
|
|
136
|
+
if (lower.includes(key)) return EVENT_TYPE_KEYWORDS[key];
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Check single-word keywords
|
|
140
|
+
const words = lower.replace(/[^\w\s]/g, '').split(/\s+/);
|
|
141
|
+
for (const word of words) {
|
|
142
|
+
if (EVENT_TYPE_KEYWORDS[word]) return EVENT_TYPE_KEYWORDS[word];
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
return undefined;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Parse a natural language query into structured retrieval filters.
|
|
150
|
+
*/
|
|
151
|
+
function parseQuery(query: string, assetId?: string): RetrievalFilters {
|
|
152
|
+
const eventType = parseEventType(query);
|
|
153
|
+
const limit = parseLimit(query);
|
|
154
|
+
const minConfidence = parseConfidence(query);
|
|
155
|
+
const timeRange = parseTimeRange(query);
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
assetId,
|
|
159
|
+
eventType,
|
|
160
|
+
minConfidence,
|
|
161
|
+
limit: limit ?? 10,
|
|
162
|
+
sortBy: 'confidence',
|
|
163
|
+
...timeRange,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// ---------------------------------------------------------------------------
|
|
168
|
+
// Formatting
|
|
169
|
+
// ---------------------------------------------------------------------------
|
|
170
|
+
|
|
171
|
+
function formatTimestamp(seconds: number): string {
|
|
172
|
+
const mins = Math.floor(seconds / 60);
|
|
173
|
+
const secs = Math.floor(seconds % 60);
|
|
174
|
+
return `${mins}:${secs.toString().padStart(2, '0')}`;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// ---------------------------------------------------------------------------
|
|
178
|
+
// Tool entry point
|
|
179
|
+
// ---------------------------------------------------------------------------
|
|
180
|
+
|
|
181
|
+
export async function run(
|
|
182
|
+
input: Record<string, unknown>,
|
|
183
|
+
_context: ToolContext,
|
|
184
|
+
): Promise<ToolExecutionResult> {
|
|
185
|
+
const query = input.query as string | undefined;
|
|
186
|
+
if (!query) {
|
|
187
|
+
return { content: 'query is required.', isError: true };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
const assetId = input.asset_id as string | undefined;
|
|
191
|
+
if (!assetId) {
|
|
192
|
+
return { content: 'asset_id is required to scope the query to a specific media asset.', isError: true };
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const filters = parseQuery(query, assetId);
|
|
196
|
+
|
|
197
|
+
// Retrieve without limit so we can apply capability filtering first, then limit
|
|
198
|
+
const userLimit = filters.limit;
|
|
199
|
+
const result = retrieveEvents({ ...filters, limit: 0 });
|
|
200
|
+
|
|
201
|
+
// Determine which capabilities are allowed based on the tracking profile
|
|
202
|
+
const profile = getTrackingProfile(assetId);
|
|
203
|
+
|
|
204
|
+
let allowedEventTypes: Set<string> | null = null;
|
|
205
|
+
const tierForEventType = new Map<string, CapabilityTier>();
|
|
206
|
+
|
|
207
|
+
if (profile) {
|
|
208
|
+
// Use the explicit profile: only enabled capabilities pass
|
|
209
|
+
allowedEventTypes = new Set<string>();
|
|
210
|
+
for (const [capName, entry] of Object.entries(profile.capabilities)) {
|
|
211
|
+
if (entry.enabled) {
|
|
212
|
+
allowedEventTypes.add(capName);
|
|
213
|
+
tierForEventType.set(capName, entry.tier);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
} else {
|
|
217
|
+
// No profile: default to 'ready' tier capabilities only
|
|
218
|
+
const readyCaps = getCapabilitiesByTier('ready');
|
|
219
|
+
if (readyCaps.length > 0) {
|
|
220
|
+
allowedEventTypes = new Set(readyCaps.map((c) => c.name));
|
|
221
|
+
for (const cap of readyCaps) {
|
|
222
|
+
tierForEventType.set(cap.name, 'ready');
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
// If no capabilities are registered at all, allow everything (pass null)
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// Filter events by allowed capabilities, then apply the user-requested limit
|
|
229
|
+
let filteredEvents = result.events;
|
|
230
|
+
if (allowedEventTypes !== null) {
|
|
231
|
+
filteredEvents = filteredEvents.filter((e) => allowedEventTypes!.has(e.eventType));
|
|
232
|
+
}
|
|
233
|
+
if (userLimit && filteredEvents.length > userLimit) {
|
|
234
|
+
filteredEvents = filteredEvents.slice(0, userLimit);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
if (filteredEvents.length === 0) {
|
|
238
|
+
const parts = ['No matching events found'];
|
|
239
|
+
if (filters.eventType) parts.push(`for event type "${filters.eventType}"`);
|
|
240
|
+
if (filters.minConfidence) parts.push(`with confidence >= ${filters.minConfidence}`);
|
|
241
|
+
if (!profile) parts.push('(defaulting to ready-tier capabilities only)');
|
|
242
|
+
parts.push(`in asset ${assetId}.`);
|
|
243
|
+
return { content: parts.join(' '), isError: false };
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const tierLabels: Record<string, string> = {
|
|
247
|
+
ready: '[Ready]',
|
|
248
|
+
beta: '[Beta]',
|
|
249
|
+
experimental: '[Experimental]',
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
const tierDisclaimers: Record<string, string> = {
|
|
253
|
+
beta: 'Beta results may have accuracy gaps.',
|
|
254
|
+
experimental: 'Experimental results are early-stage; expect noise.',
|
|
255
|
+
};
|
|
256
|
+
|
|
257
|
+
const eventSummaries = filteredEvents.map((e, i) => {
|
|
258
|
+
const tier = tierForEventType.get(e.eventType) ?? getCapabilityByName(e.eventType)?.tier;
|
|
259
|
+
const tierLabel = tier ? tierLabels[tier] ?? `[${tier}]` : '[Ready]';
|
|
260
|
+
const disclaimer = tier ? tierDisclaimers[tier] : undefined;
|
|
261
|
+
|
|
262
|
+
return {
|
|
263
|
+
rank: i + 1,
|
|
264
|
+
id: e.id,
|
|
265
|
+
eventType: e.eventType,
|
|
266
|
+
tierLabel,
|
|
267
|
+
...(disclaimer ? { confidenceDisclaimer: disclaimer } : {}),
|
|
268
|
+
timeRange: `${formatTimestamp(e.startTime)} – ${formatTimestamp(e.endTime)}`,
|
|
269
|
+
startTime: e.startTime,
|
|
270
|
+
endTime: e.endTime,
|
|
271
|
+
confidence: Math.round(e.confidence * 100) / 100,
|
|
272
|
+
reasons: e.reasons,
|
|
273
|
+
metadata: e.metadata,
|
|
274
|
+
};
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
// Collect disclaimers for any non-ready tiers present in results
|
|
278
|
+
const activeTiers = new Set(eventSummaries.map((e) => e.tierLabel));
|
|
279
|
+
const disclaimers: string[] = [];
|
|
280
|
+
if (activeTiers.has('[Beta]')) disclaimers.push(tierDisclaimers.beta);
|
|
281
|
+
if (activeTiers.has('[Experimental]')) disclaimers.push(tierDisclaimers.experimental);
|
|
282
|
+
|
|
283
|
+
return {
|
|
284
|
+
content: JSON.stringify({
|
|
285
|
+
query,
|
|
286
|
+
parsedFilters: {
|
|
287
|
+
eventType: filters.eventType ?? null,
|
|
288
|
+
minConfidence: filters.minConfidence ?? null,
|
|
289
|
+
limit: filters.limit,
|
|
290
|
+
startTimeMin: filters.startTimeMin ?? null,
|
|
291
|
+
startTimeMax: filters.startTimeMax ?? null,
|
|
292
|
+
},
|
|
293
|
+
trackingProfile: profile ? 'custom' : 'default (ready tier only)',
|
|
294
|
+
...(disclaimers.length > 0 ? { disclaimers } : {}),
|
|
295
|
+
totalResults: eventSummaries.length,
|
|
296
|
+
events: eventSummaries,
|
|
297
|
+
}, null, 2),
|
|
298
|
+
isError: false,
|
|
299
|
+
};
|
|
300
|
+
}
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recalibration tool for media event detection.
|
|
3
|
+
*
|
|
4
|
+
* Reads all feedback for an asset, analyzes correction patterns, and
|
|
5
|
+
* re-ranks existing events using updated heuristics. Updates confidence
|
|
6
|
+
* scores in the media_events table.
|
|
7
|
+
*
|
|
8
|
+
* All interfaces are generic — works for any event type.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
|
|
12
|
+
import { getFeedbackForAsset, type EventFeedback } from '../services/feedback-store.js';
|
|
13
|
+
import { aggregateFeedback } from '../services/feedback-aggregation.js';
|
|
14
|
+
import { getEventsForAsset, getMediaAssetById } from '../../../../memory/media-store.js';
|
|
15
|
+
import { getDb } from '../../../../memory/db.js';
|
|
16
|
+
import { mediaEvents } from '../../../../memory/schema.js';
|
|
17
|
+
import { eq } from 'drizzle-orm';
|
|
18
|
+
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
// Types
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
|
|
23
|
+
interface RecalibrationAdjustment {
|
|
24
|
+
eventType: string;
|
|
25
|
+
action: string;
|
|
26
|
+
detail: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface EventUpdate {
|
|
30
|
+
eventId: string;
|
|
31
|
+
eventType: string;
|
|
32
|
+
oldConfidence: number;
|
|
33
|
+
newConfidence: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// ---------------------------------------------------------------------------
|
|
37
|
+
// Tool entry point
|
|
38
|
+
// ---------------------------------------------------------------------------
|
|
39
|
+
|
|
40
|
+
export async function run(
|
|
41
|
+
input: Record<string, unknown>,
|
|
42
|
+
context: ToolContext,
|
|
43
|
+
): Promise<ToolExecutionResult> {
|
|
44
|
+
const assetId = input.asset_id as string | undefined;
|
|
45
|
+
if (!assetId) {
|
|
46
|
+
return { content: 'asset_id is required.', isError: true };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const asset = getMediaAssetById(assetId);
|
|
50
|
+
if (!asset) {
|
|
51
|
+
return { content: `Asset "${assetId}" not found.`, isError: true };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const allFeedback = getFeedbackForAsset(assetId);
|
|
55
|
+
if (allFeedback.length === 0) {
|
|
56
|
+
return {
|
|
57
|
+
content: JSON.stringify({
|
|
58
|
+
message: 'No feedback found for this asset. Submit feedback first, then recalibrate.',
|
|
59
|
+
assetId,
|
|
60
|
+
}, null, 2),
|
|
61
|
+
isError: false,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const aggregation = aggregateFeedback(assetId);
|
|
66
|
+
const allEvents = getEventsForAsset(assetId);
|
|
67
|
+
|
|
68
|
+
// Build a map of event ID to its feedback entries
|
|
69
|
+
const feedbackByEventId = new Map<string, EventFeedback[]>();
|
|
70
|
+
for (const fb of allFeedback) {
|
|
71
|
+
if (!feedbackByEventId.has(fb.eventId)) {
|
|
72
|
+
feedbackByEventId.set(fb.eventId, []);
|
|
73
|
+
}
|
|
74
|
+
feedbackByEventId.get(fb.eventId)!.push(fb);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Build a map of event ID to event type for filtering feedback by type
|
|
78
|
+
const eventTypeById = new Map<string, string>();
|
|
79
|
+
for (const ev of allEvents) {
|
|
80
|
+
eventTypeById.set(ev.id, ev.eventType);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const adjustments: RecalibrationAdjustment[] = [];
|
|
84
|
+
const eventUpdates: EventUpdate[] = [];
|
|
85
|
+
const db = getDb();
|
|
86
|
+
|
|
87
|
+
// Analyze patterns per event type
|
|
88
|
+
for (const stats of aggregation.statsByEventType) {
|
|
89
|
+
const { eventType, correct, incorrect, boundaryEdit, missed } = stats;
|
|
90
|
+
const totalReviewed = correct + incorrect + boundaryEdit + missed;
|
|
91
|
+
if (totalReviewed === 0) continue;
|
|
92
|
+
|
|
93
|
+
// Pattern 1: High false positive rate — penalize low-confidence events
|
|
94
|
+
const falsePositiveRate = totalReviewed > 0 ? incorrect / totalReviewed : 0;
|
|
95
|
+
if (falsePositiveRate > 0.3 && incorrect >= 2) {
|
|
96
|
+
adjustments.push({
|
|
97
|
+
eventType,
|
|
98
|
+
action: 'penalize_low_confidence',
|
|
99
|
+
detail: `False positive rate ${(falsePositiveRate * 100).toFixed(1)}% (${incorrect}/${totalReviewed}) — reducing confidence on unreviewed events of this type`,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Pattern 2: Many missed events — note for threshold adjustment
|
|
104
|
+
if (missed >= 2) {
|
|
105
|
+
adjustments.push({
|
|
106
|
+
eventType,
|
|
107
|
+
action: 'note_missed_events',
|
|
108
|
+
detail: `${missed} missed events reported — consider lowering detection threshold or adding detection rules for this type`,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Pattern 3: Boundary edits — compute average adjustment
|
|
113
|
+
if (boundaryEdit >= 1) {
|
|
114
|
+
const boundaryFeedback = allFeedback.filter(
|
|
115
|
+
(fb) => fb.feedbackType === 'boundary_edit' && eventTypeById.get(fb.eventId) === eventType,
|
|
116
|
+
);
|
|
117
|
+
let startAdjTotal = 0;
|
|
118
|
+
let endAdjTotal = 0;
|
|
119
|
+
let startAdjCount = 0;
|
|
120
|
+
let endAdjCount = 0;
|
|
121
|
+
|
|
122
|
+
for (const fb of boundaryFeedback) {
|
|
123
|
+
if (fb.originalStartTime !== null && fb.correctedStartTime !== null) {
|
|
124
|
+
startAdjTotal += fb.correctedStartTime - fb.originalStartTime;
|
|
125
|
+
startAdjCount++;
|
|
126
|
+
}
|
|
127
|
+
if (fb.originalEndTime !== null && fb.correctedEndTime !== null) {
|
|
128
|
+
endAdjTotal += fb.correctedEndTime - fb.originalEndTime;
|
|
129
|
+
endAdjCount++;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const avgStartAdj = startAdjCount > 0 ? startAdjTotal / startAdjCount : 0;
|
|
134
|
+
const avgEndAdj = endAdjCount > 0 ? endAdjTotal / endAdjCount : 0;
|
|
135
|
+
|
|
136
|
+
if (startAdjCount > 0 || endAdjCount > 0) {
|
|
137
|
+
adjustments.push({
|
|
138
|
+
eventType,
|
|
139
|
+
action: 'boundary_correction_pattern',
|
|
140
|
+
detail: `Average boundary adjustment: start ${avgStartAdj >= 0 ? '+' : ''}${avgStartAdj.toFixed(2)}s (n=${startAdjCount}), end ${avgEndAdj >= 0 ? '+' : ''}${avgEndAdj.toFixed(2)}s (n=${endAdjCount})`,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Re-rank events: adjust confidence based on feedback
|
|
147
|
+
for (const event of allEvents) {
|
|
148
|
+
const eventFeedback = feedbackByEventId.get(event.id);
|
|
149
|
+
if (!eventFeedback || eventFeedback.length === 0) {
|
|
150
|
+
// For events without direct feedback, apply type-level adjustments
|
|
151
|
+
const stats = aggregation.statsByEventType.find((s) => s.eventType === event.eventType);
|
|
152
|
+
if (stats) {
|
|
153
|
+
const totalReviewed = stats.correct + stats.incorrect + stats.boundaryEdit + stats.missed;
|
|
154
|
+
const falsePositiveRate = totalReviewed > 0 ? stats.incorrect / totalReviewed : 0;
|
|
155
|
+
|
|
156
|
+
// If high false positive rate for this type, reduce unreviewed event confidence
|
|
157
|
+
if (falsePositiveRate > 0.3 && totalReviewed >= 3) {
|
|
158
|
+
const penalty = Math.min(falsePositiveRate * 0.3, 0.2);
|
|
159
|
+
const newConfidence = Math.max(0.05, event.confidence - penalty);
|
|
160
|
+
if (newConfidence !== event.confidence) {
|
|
161
|
+
db.update(mediaEvents)
|
|
162
|
+
.set({ confidence: newConfidence })
|
|
163
|
+
.where(eq(mediaEvents.id, event.id))
|
|
164
|
+
.run();
|
|
165
|
+
eventUpdates.push({
|
|
166
|
+
eventId: event.id,
|
|
167
|
+
eventType: event.eventType,
|
|
168
|
+
oldConfidence: event.confidence,
|
|
169
|
+
newConfidence,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Direct feedback: adjust confidence based on the latest feedback
|
|
178
|
+
const latestFeedback = eventFeedback.sort((a, b) => b.createdAt - a.createdAt)[0];
|
|
179
|
+
let newConfidence = event.confidence;
|
|
180
|
+
|
|
181
|
+
switch (latestFeedback.feedbackType) {
|
|
182
|
+
case 'correct':
|
|
183
|
+
// Boost confidence toward 1.0
|
|
184
|
+
newConfidence = Math.min(1.0, event.confidence + (1.0 - event.confidence) * 0.3);
|
|
185
|
+
break;
|
|
186
|
+
case 'incorrect':
|
|
187
|
+
// Sharply reduce confidence
|
|
188
|
+
newConfidence = Math.max(0.05, event.confidence * 0.3);
|
|
189
|
+
break;
|
|
190
|
+
case 'boundary_edit':
|
|
191
|
+
// Slight confidence boost (event was real but boundaries were off)
|
|
192
|
+
newConfidence = Math.min(1.0, event.confidence + (1.0 - event.confidence) * 0.15);
|
|
193
|
+
break;
|
|
194
|
+
case 'missed':
|
|
195
|
+
// User-reported events keep their initial confidence
|
|
196
|
+
break;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
newConfidence = Math.round(newConfidence * 1000) / 1000;
|
|
200
|
+
|
|
201
|
+
if (newConfidence !== event.confidence) {
|
|
202
|
+
db.update(mediaEvents)
|
|
203
|
+
.set({ confidence: newConfidence })
|
|
204
|
+
.where(eq(mediaEvents.id, event.id))
|
|
205
|
+
.run();
|
|
206
|
+
eventUpdates.push({
|
|
207
|
+
eventId: event.id,
|
|
208
|
+
eventType: event.eventType,
|
|
209
|
+
oldConfidence: event.confidence,
|
|
210
|
+
newConfidence,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
context.onOutput?.(`Recalibrated ${eventUpdates.length} events based on ${allFeedback.length} feedback entries.\n`);
|
|
216
|
+
|
|
217
|
+
return {
|
|
218
|
+
content: JSON.stringify({
|
|
219
|
+
message: `Recalibration complete for asset ${assetId}`,
|
|
220
|
+
assetId,
|
|
221
|
+
totalFeedbackEntries: allFeedback.length,
|
|
222
|
+
adjustments,
|
|
223
|
+
eventsUpdated: eventUpdates.length,
|
|
224
|
+
eventUpdates: eventUpdates.map((u) => ({
|
|
225
|
+
eventId: u.eventId,
|
|
226
|
+
eventType: u.eventType,
|
|
227
|
+
oldConfidence: u.oldConfidence,
|
|
228
|
+
newConfidence: u.newConfidence,
|
|
229
|
+
delta: Math.round((u.newConfidence - u.oldConfidence) * 1000) / 1000,
|
|
230
|
+
})),
|
|
231
|
+
aggregation: aggregation.statsByEventType,
|
|
232
|
+
}, null, 2),
|
|
233
|
+
isError: false,
|
|
234
|
+
};
|
|
235
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Select and persist a tracking profile for a media asset.
|
|
3
|
+
*
|
|
4
|
+
* When called without capabilities, returns the available capabilities
|
|
5
|
+
* organized by tier so the user can choose. When called with capabilities,
|
|
6
|
+
* validates them against the registry and stores the profile.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
|
|
10
|
+
import { getMediaAssetById, setTrackingProfile, getTrackingProfile, type CapabilityProfile } from '../../../../memory/media-store.js';
|
|
11
|
+
import {
|
|
12
|
+
getCapabilities,
|
|
13
|
+
getCapabilityByName,
|
|
14
|
+
getRegisteredDomains,
|
|
15
|
+
type Capability,
|
|
16
|
+
} from '../services/capability-registry.js';
|
|
17
|
+
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
// Helpers
|
|
20
|
+
// ---------------------------------------------------------------------------
|
|
21
|
+
|
|
22
|
+
function groupByTier(caps: Capability[]): Record<string, Capability[]> {
|
|
23
|
+
const groups: Record<string, Capability[]> = { ready: [], beta: [], experimental: [] };
|
|
24
|
+
for (const cap of caps) {
|
|
25
|
+
if (!groups[cap.tier]) groups[cap.tier] = [];
|
|
26
|
+
groups[cap.tier].push(cap);
|
|
27
|
+
}
|
|
28
|
+
return groups;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
32
|
+
// Tool entry point
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
|
|
35
|
+
export async function run(
|
|
36
|
+
input: Record<string, unknown>,
|
|
37
|
+
_context: ToolContext,
|
|
38
|
+
): Promise<ToolExecutionResult> {
|
|
39
|
+
const assetId = input.asset_id as string | undefined;
|
|
40
|
+
if (!assetId) {
|
|
41
|
+
return { content: 'asset_id is required.', isError: true };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const asset = getMediaAssetById(assetId);
|
|
45
|
+
if (!asset) {
|
|
46
|
+
return { content: `No media asset found with id "${assetId}".`, isError: true };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const requestedCapabilities = input.capabilities as string[] | undefined;
|
|
50
|
+
|
|
51
|
+
// If no capabilities provided, return available options for the user to choose
|
|
52
|
+
if (!requestedCapabilities || requestedCapabilities.length === 0) {
|
|
53
|
+
const allCaps = getCapabilities();
|
|
54
|
+
const domains = getRegisteredDomains();
|
|
55
|
+
const byTier = groupByTier(allCaps);
|
|
56
|
+
|
|
57
|
+
const currentProfile = getTrackingProfile(assetId);
|
|
58
|
+
|
|
59
|
+
return {
|
|
60
|
+
content: JSON.stringify({
|
|
61
|
+
message: 'No capabilities specified. Here are the available capabilities organized by tier. Call again with a capabilities array to enable specific ones.',
|
|
62
|
+
domains,
|
|
63
|
+
availableCapabilities: {
|
|
64
|
+
ready: byTier.ready.map((c) => ({
|
|
65
|
+
name: c.name,
|
|
66
|
+
description: c.description,
|
|
67
|
+
domain: c.domain,
|
|
68
|
+
granularity: c.granularity ?? null,
|
|
69
|
+
})),
|
|
70
|
+
beta: byTier.beta.map((c) => ({
|
|
71
|
+
name: c.name,
|
|
72
|
+
description: c.description,
|
|
73
|
+
domain: c.domain,
|
|
74
|
+
granularity: c.granularity ?? null,
|
|
75
|
+
note: 'Beta: functional but may have accuracy gaps',
|
|
76
|
+
})),
|
|
77
|
+
experimental: byTier.experimental.map((c) => ({
|
|
78
|
+
name: c.name,
|
|
79
|
+
description: c.description,
|
|
80
|
+
domain: c.domain,
|
|
81
|
+
granularity: c.granularity ?? null,
|
|
82
|
+
note: 'Experimental: early-stage, expect noise in results',
|
|
83
|
+
})),
|
|
84
|
+
},
|
|
85
|
+
currentProfile: currentProfile ? currentProfile.capabilities : null,
|
|
86
|
+
}, null, 2),
|
|
87
|
+
isError: false,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Validate requested capabilities exist in the registry
|
|
92
|
+
const capabilities: CapabilityProfile = {};
|
|
93
|
+
const unknownCaps: string[] = [];
|
|
94
|
+
|
|
95
|
+
for (const capName of requestedCapabilities) {
|
|
96
|
+
const registered = getCapabilityByName(capName);
|
|
97
|
+
if (!registered) {
|
|
98
|
+
unknownCaps.push(capName);
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
capabilities[capName] = { enabled: true, tier: registered.tier };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (unknownCaps.length > 0) {
|
|
105
|
+
const allCaps = getCapabilities();
|
|
106
|
+
return {
|
|
107
|
+
content: JSON.stringify({
|
|
108
|
+
error: `Unknown capabilities: ${unknownCaps.join(', ')}`,
|
|
109
|
+
availableCapabilities: allCaps.map((c) => c.name),
|
|
110
|
+
}, null, 2),
|
|
111
|
+
isError: true,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Store the profile
|
|
116
|
+
const profile = setTrackingProfile(assetId, capabilities);
|
|
117
|
+
|
|
118
|
+
// Build response with tier labels
|
|
119
|
+
const profileSummary: Record<string, { enabled: boolean; tier: string; tierLabel: string }> = {};
|
|
120
|
+
for (const [name, entry] of Object.entries(profile.capabilities)) {
|
|
121
|
+
const tierLabels: Record<string, string> = {
|
|
122
|
+
ready: '[Ready]',
|
|
123
|
+
beta: '[Beta]',
|
|
124
|
+
experimental: '[Experimental]',
|
|
125
|
+
};
|
|
126
|
+
profileSummary[name] = {
|
|
127
|
+
enabled: entry.enabled,
|
|
128
|
+
tier: entry.tier,
|
|
129
|
+
tierLabel: tierLabels[entry.tier] ?? `[${entry.tier}]`,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return {
|
|
134
|
+
content: JSON.stringify({
|
|
135
|
+
message: 'Tracking profile saved.',
|
|
136
|
+
assetId: profile.assetId,
|
|
137
|
+
profileId: profile.id,
|
|
138
|
+
capabilities: profileSummary,
|
|
139
|
+
}, null, 2),
|
|
140
|
+
isError: false,
|
|
141
|
+
};
|
|
142
|
+
}
|