@larkup/tool-video-intelligence 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +150 -0
- package/LICENSE +176 -0
- package/README.md +281 -0
- package/compose.gpu.yaml +14 -0
- package/compose.yaml +71 -0
- package/dist/agent.d.ts +131 -0
- package/dist/agent.js +2087 -0
- package/dist/brief.d.ts +2 -0
- package/dist/brief.js +37 -0
- package/dist/client.d.ts +46 -0
- package/dist/client.js +139 -0
- package/dist/contracts.d.ts +331 -0
- package/dist/contracts.js +1 -0
- package/dist/index.d.ts +87 -0
- package/dist/index.js +391 -0
- package/dist/runtime.d.ts +96 -0
- package/dist/runtime.js +592 -0
- package/dist/ui.d.ts +82 -0
- package/dist/ui.js +87 -0
- package/package.json +84 -0
- package/runtime/Dockerfile +119 -0
- package/runtime/app/__init__.py +3 -0
- package/runtime/app/__main__.py +19 -0
- package/runtime/app/api/__init__.py +0 -0
- package/runtime/app/api/deps.py +69 -0
- package/runtime/app/api/v1.py +166 -0
- package/runtime/app/config.py +78 -0
- package/runtime/app/db/__init__.py +0 -0
- package/runtime/app/db/schemas.py +162 -0
- package/runtime/app/db/store.py +466 -0
- package/runtime/app/main.py +27 -0
- package/runtime/app/model_configuration.py +157 -0
- package/runtime/app/services/__init__.py +0 -0
- package/runtime/app/services/brain.py +2221 -0
- package/runtime/app/services/embedding.py +473 -0
- package/runtime/app/services/jobs.py +237 -0
- package/runtime/app/services/motion.py +66 -0
- package/runtime/app/services/pipeline.py +1911 -0
- package/runtime/app/services/scene.py +161 -0
- package/runtime/app/services/storage.py +44 -0
- package/runtime/app/services/transcription.py +667 -0
- package/runtime/app/services/vision.py +1441 -0
- package/runtime/app/utils/__init__.py +0 -0
- package/runtime/app/utils/timing.py +99 -0
- package/runtime/app/worker.py +20 -0
- package/runtime/pyproject.toml +56 -0
- package/runtime/requirements-cpu.txt +15 -0
- package/runtime/requirements-smoke.txt +7 -0
- package/runtime/requirements.txt +14 -0
- package/runtime/uv.lock +3637 -0
- package/scripts/grant-cloud-credits.sh +43 -0
- package/scripts/runtime.mjs +156 -0
- package/scripts/validate-indexing.mjs +168 -0
- package/tool.manifest.json +617 -0
package/dist/agent.js
ADDED
|
@@ -0,0 +1,2087 @@
|
|
|
1
|
+
import { trackUsageEvent } from '@larkup/core/analytics-store';
|
|
2
|
+
const MAX_INSPECTION_CHUNK_SECS = 60;
|
|
3
|
+
const OUTCOME_RESOLUTION_WINDOW_SECS = 30;
|
|
4
|
+
/** How much source a single targeted look covers on each side of a located moment. */
|
|
5
|
+
const TARGET_PADDING_SECS = 8;
|
|
6
|
+
/** Independent looks dispatched together before the next wave starts. */
|
|
7
|
+
const MAX_PARALLEL_INSPECTIONS = 4;
|
|
8
|
+
/** The whole evidence-query fallback shares one deadline, including re-watch. */
|
|
9
|
+
const INTERACTIVE_INSPECTION_BUDGET_MS = 45_000;
|
|
10
|
+
/** Leave part of the shared deadline for a provider-backed inspection if needed. */
|
|
11
|
+
const INTERACTIVE_REWATCH_BUDGET_MS = 20_000;
|
|
12
|
+
const TRANSITION_LANGUAGE = /chang|updat|finish|final|result|conclud|resolv|settl/i;
|
|
13
|
+
const HUMAN_ROLE_LANGUAGE = /\b(?:person|people|individual|man|woman|participant|contestant|player|member|speaker|presenter|host|guest|attendee)\b/i;
|
|
14
|
+
const GENERIC_IDENTITY_LANGUAGE = /^(?:(?:unknown|unidentified|unnamed)(?:\s+(?:person|people|participant|contestant|player|member|speaker|presenter|host|guest|attendee|individual|man|woman))?|person|people|participant|contestant|player|member|speaker|presenter|host|guest|attendee|individual|man|woman|team|group|studio participants?)(?:\s+\d+)?$/i;
|
|
15
|
+
const ATTRIBUTE_ACTION_LANGUAGE = /\b(?:wear\w*|dress\w*|clothing|clothes|outfit|shirt|jersey|jacket|coat|trousers|pants|skirt|shoe\w*|hat|hold\w*|carr\w*|stand\w*|sit\w*|driv\w*|eat\w*|drink\w*)\b|(?:يرتدي|لابس|ملابس|قميص|تيشيرت|جاكيت|بنطلون|حذاء|قبعة|يمسك|يحمل|يقف|يجلس|يقود|يأكل|يشرب)/iu;
|
|
16
|
+
const LIMITATION_LANGUAGE = /\b(?:no explicit|not explicit|does not explicit|not (?:a )?complete|not established|not shown|unclear|unknown|unresolved|incomplete)\b/i;
|
|
17
|
+
/** Chat actions stay inside the installed tool; the host only sees workflow roles. */
|
|
18
|
+
export const AGENT_TOOLS = [
|
|
19
|
+
{
|
|
20
|
+
name: 'queryVideoEvidence',
|
|
21
|
+
description: 'Answer a question about an indexed video. Always use the existing RAG index first; perform a bounded direct re-watch only when the retrieved evidence is genuinely incomplete or conflicting. For requests that explicitly ask for every item, the full account, or everything said, set exhaustive=true and follow continuation.nextCursor until hasMore=false.',
|
|
22
|
+
parameters: {
|
|
23
|
+
type: 'object',
|
|
24
|
+
additionalProperties: false,
|
|
25
|
+
required: ['mediaAssetId', 'query'],
|
|
26
|
+
properties: {
|
|
27
|
+
mediaAssetId: { type: 'string', minLength: 1 },
|
|
28
|
+
query: { type: 'string', minLength: 1, maxLength: 2_000 },
|
|
29
|
+
limit: { type: 'integer', minimum: 1, maximum: 48 },
|
|
30
|
+
exhaustive: { type: 'boolean' },
|
|
31
|
+
cursor: { type: 'integer', minimum: 0 },
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
method: 'queryVideoEvidence',
|
|
35
|
+
workflow: 'evidence-query',
|
|
36
|
+
evidenceInput: 'media-asset',
|
|
37
|
+
systemPromptFragment: 'Use this action for media questions. It reads the existing indexed evidence first and only watches a bounded source moment when that evidence is incomplete or conflicting. Never trigger extra analysis merely to restate an answer the returned evidence already establishes. For an explicit every/all/complete-source request, call with exhaustive=true and keep calling with continuation.nextCursor until hasMore=false; do not mistake one top-K page for the complete answer. Never say you do not know or that the source lacks an answer before this action has attempted its available fallback. State an outcome, final result, identity, count, or exact visible fact only when established. Answer naturally as someone who watched and remembers the material: lead with the answer itself (for example, "X won" or "He wore Y"), not phrases such as "the video shows", "the analysis indicates", or "according to the retrieved evidence". Never mention retrieval, search, indexing, frames, tools, or analysis unless the user asks how the answer was found.',
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
name: 'inspectVideoKnowledge',
|
|
41
|
+
description: 'Inspect a bounded source range when another evidence action asks for corroboration. Do not scan an entire video.',
|
|
42
|
+
parameters: {
|
|
43
|
+
type: 'object',
|
|
44
|
+
additionalProperties: false,
|
|
45
|
+
required: ['mediaAssetId', 'startSecs', 'endSecs', 'purpose', 'queryId'],
|
|
46
|
+
properties: {
|
|
47
|
+
mediaAssetId: { type: 'string' },
|
|
48
|
+
startSecs: { type: 'number', minimum: 0 },
|
|
49
|
+
endSecs: { type: 'number', minimum: 0 },
|
|
50
|
+
purpose: {
|
|
51
|
+
type: 'string',
|
|
52
|
+
enum: ['verify-visual', 'high-res-ocr', 'compare', 'count', 'track', 'code'],
|
|
53
|
+
},
|
|
54
|
+
queryId: { type: 'string', minLength: 1, maxLength: 128 },
|
|
55
|
+
question: { type: 'string', minLength: 1, maxLength: 2000 },
|
|
56
|
+
maxFrames: { type: 'integer', minimum: 1, maximum: 24 },
|
|
57
|
+
continuousSequence: { type: 'boolean' },
|
|
58
|
+
includeSpeech: { type: 'boolean' },
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
method: 'inspectVideoKnowledge',
|
|
62
|
+
workflow: 'evidence-refinement',
|
|
63
|
+
systemPromptFragment: 'When this action returns fresh evidence, query the relevant evidence source again before making the claim.',
|
|
64
|
+
},
|
|
65
|
+
];
|
|
66
|
+
export function attachVideoIntelligenceAgentClient(client, fetcher = globalThis.fetch) {
|
|
67
|
+
const inspectVideoKnowledge = createInspector(fetcher);
|
|
68
|
+
return Object.assign(client, {
|
|
69
|
+
inspectVideoKnowledge,
|
|
70
|
+
async queryVideoEvidence(input, context) {
|
|
71
|
+
if (!isValidQueryInput(input))
|
|
72
|
+
return { success: false, error: 'A media asset and question are required.' };
|
|
73
|
+
const mediaEvidence = context.mediaEvidence;
|
|
74
|
+
if (!mediaEvidence)
|
|
75
|
+
return { success: false, error: 'The host has not provided scoped media evidence access.' };
|
|
76
|
+
const startedAt = Date.now();
|
|
77
|
+
const asset = await mediaEvidence.getAsset(input.mediaAssetId);
|
|
78
|
+
if (!asset || asset.processingStatus !== 'completed')
|
|
79
|
+
return { success: false, error: 'A completed video asset is required.' };
|
|
80
|
+
const focusedQuery = focusQuestion(input.query, asset.fileName);
|
|
81
|
+
const plan = mediaEvidence.planQuestion(focusedQuery);
|
|
82
|
+
const focusedInput = {
|
|
83
|
+
...input,
|
|
84
|
+
query: focusedQuery,
|
|
85
|
+
// Requests that span a whole recording need deterministic pagination
|
|
86
|
+
// through the index. Callers should not need to know this tool's query
|
|
87
|
+
// planner well enough to opt into exhaustive retrieval themselves.
|
|
88
|
+
exhaustive: input.exhaustive ??
|
|
89
|
+
(plan.kinds.some((kind) => kind === 'question-inventory' ||
|
|
90
|
+
kind === 'source-inventory' ||
|
|
91
|
+
kind === 'entity-inventory') ||
|
|
92
|
+
(plan.requiresBroadCoverage === true && !plan.kinds.includes('evaluation'))),
|
|
93
|
+
};
|
|
94
|
+
const durationSecs = asset.durationSecs;
|
|
95
|
+
// Start with the index alone. Planning and visual locating are useful
|
|
96
|
+
// fallbacks, but waiting for both before checking an already-complete RAG
|
|
97
|
+
// answer made simple questions feel like analysis jobs.
|
|
98
|
+
let hits = await retrieve(mediaEvidence, focusedInput, plan, durationSecs);
|
|
99
|
+
let investigation;
|
|
100
|
+
let locatedRanges = [];
|
|
101
|
+
let assessment = assessEvidence(hits, focusedInput.query, plan, durationSecs, undefined);
|
|
102
|
+
if (!assessment.sufficient) {
|
|
103
|
+
[investigation, locatedRanges] = await Promise.all([
|
|
104
|
+
mediaEvidence.planInvestigation?.(input.mediaAssetId, focusedInput.query),
|
|
105
|
+
// Where independent indexed signals agree the answer is. This is
|
|
106
|
+
// measured navigation, not evidence, and only runs after the fast
|
|
107
|
+
// indexed answer test has failed.
|
|
108
|
+
mediaEvidence.locate?.(input.mediaAssetId, focusedInput.query, {
|
|
109
|
+
maxRanges: 4,
|
|
110
|
+
maxWindowSecs: MAX_INSPECTION_CHUNK_SECS,
|
|
111
|
+
}) ?? Promise.resolve([]),
|
|
112
|
+
]);
|
|
113
|
+
const locatedEvidence = await retrieveEvidenceInRanges(mediaEvidence, focusedInput, plan, durationSecs, locatedRanges);
|
|
114
|
+
hits = mergeHits(locatedEvidence, hits);
|
|
115
|
+
assessment = assessEvidence(hits, focusedInput.query, plan, durationSecs, investigation);
|
|
116
|
+
}
|
|
117
|
+
let analyzedRanges = [];
|
|
118
|
+
let inspection;
|
|
119
|
+
let analysisUnavailable;
|
|
120
|
+
const responseDeadline = startedAt + INTERACTIVE_INSPECTION_BUDGET_MS;
|
|
121
|
+
// Retrieval locates; it does not decide that something is unknowable.
|
|
122
|
+
// When the index cannot answer, the agent goes and watches the moments
|
|
123
|
+
// the index pointed at rather than reporting that the source is silent.
|
|
124
|
+
if (!assessment.sufficient && plan.requiresInspectionWhenInsufficient) {
|
|
125
|
+
const targets = inspectionTargets(hits, focusedInput.query, plan, assessment, durationSecs, investigation, locatedRanges);
|
|
126
|
+
// Re-reading the source directly is the cheap way to settle this, so it
|
|
127
|
+
// goes first. Dispatching a re-index below costs minutes on a cold
|
|
128
|
+
// worker and frequently outlives the turn, which is what turns an
|
|
129
|
+
// answerable question into an unconfirmed one.
|
|
130
|
+
if (targets.length > 0 && mediaEvidence.reWatch) {
|
|
131
|
+
const readings = await mediaEvidence
|
|
132
|
+
.reWatch(input.mediaAssetId, focusedInput.query,
|
|
133
|
+
// The host reads these windows together, so a fourth costs
|
|
134
|
+
// little wall time and a progression usually needs more than two.
|
|
135
|
+
targets.slice(0, 4).map((range) => ({
|
|
136
|
+
...range,
|
|
137
|
+
lookingFor: windowObjective(plan),
|
|
138
|
+
})), {
|
|
139
|
+
maxWaitMs: Math.max(1_000, Math.min(INTERACTIVE_REWATCH_BUDGET_MS, responseDeadline - Date.now())),
|
|
140
|
+
knownEntities: inspectionEntityHints(hits, plan),
|
|
141
|
+
})
|
|
142
|
+
.catch(() => []);
|
|
143
|
+
const established = readings.filter((reading) => reading.settlesQuestion && reading.found.trim());
|
|
144
|
+
if (established.length > 0) {
|
|
145
|
+
const supportingClip = established[0].range;
|
|
146
|
+
const indexedEvidence = evidenceHitsFor(hits, assessment, plan, focusedInput, durationSecs);
|
|
147
|
+
return {
|
|
148
|
+
success: true,
|
|
149
|
+
mediaAssetId: input.mediaAssetId,
|
|
150
|
+
fileName: asset.fileName,
|
|
151
|
+
evidence: indexedEvidence,
|
|
152
|
+
supportingClip,
|
|
153
|
+
ui: citationSurface(asset, context, established.map((reading) => ({
|
|
154
|
+
modality: 'visual',
|
|
155
|
+
timeRange: { ...reading.range, precision: 'estimated' },
|
|
156
|
+
})), supportingClip.startSecs),
|
|
157
|
+
directObservation: {
|
|
158
|
+
readings: established,
|
|
159
|
+
rule: 'These are direct readings taken by re-watching the source just now, for this ' +
|
|
160
|
+
'question. Answer from them and cite the timestamps they came from, and do not ' +
|
|
161
|
+
'report that the source fails to show something a reading here establishes. ' +
|
|
162
|
+
'Weigh them against each other and against the indexed evidence rather than ' +
|
|
163
|
+
'taking each at face value: a reading is strongest where several readings and ' +
|
|
164
|
+
'the index agree. A lone reading that contradicts both the other readings and ' +
|
|
165
|
+
'the index -- naming someone or something that appears nowhere else -- is a ' +
|
|
166
|
+
'misreading, whatever confidence it carries; leave it out rather than reporting ' +
|
|
167
|
+
'it. Where the readings genuinely conflict, say what is certain and what is not.',
|
|
168
|
+
},
|
|
169
|
+
...temporalContextResult(hits, plan),
|
|
170
|
+
claimVerification: {
|
|
171
|
+
status: 'directly-established',
|
|
172
|
+
directlyEstablished: true,
|
|
173
|
+
rule: 'The source was re-watched for this question; answer from what it established.',
|
|
174
|
+
},
|
|
175
|
+
investigation: {
|
|
176
|
+
answerPath: 'rag+rewatch',
|
|
177
|
+
responseTimeMs: Date.now() - startedAt,
|
|
178
|
+
analyzedRanges: established.map((reading) => reading.range),
|
|
179
|
+
broadCoverage: plan.requiresBroadCoverage === true,
|
|
180
|
+
coverage: investigation?.coverage,
|
|
181
|
+
},
|
|
182
|
+
...(focusedInput.exhaustive
|
|
183
|
+
? { continuation: exhaustiveContinuation(hits, assessment, focusedInput) }
|
|
184
|
+
: {}),
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
if (targets.length > 0) {
|
|
189
|
+
const outcome = await inspectRanges({
|
|
190
|
+
targets,
|
|
191
|
+
input: focusedInput,
|
|
192
|
+
plan,
|
|
193
|
+
context,
|
|
194
|
+
inspectVideoKnowledge,
|
|
195
|
+
knownEntities: inspectionEntityHints(hits, plan),
|
|
196
|
+
deadline: responseDeadline,
|
|
197
|
+
onWaveComplete: async (watched) => {
|
|
198
|
+
// An inspection appends evidence asynchronously to the same
|
|
199
|
+
// index used by normal RAG. A broad outcome lookup can still
|
|
200
|
+
// rank an older overview ahead of that new record, especially
|
|
201
|
+
// when it expands into a source-wide timeline. Read each
|
|
202
|
+
// range we just watched explicitly first, then merge it with
|
|
203
|
+
// normal retrieval. This is provenance-based rather than
|
|
204
|
+
// question-specific: every live inspection must be able to
|
|
205
|
+
// contribute its own observations to the answer.
|
|
206
|
+
const [freshWatchedEvidence, retrieved] = await Promise.all([
|
|
207
|
+
retrieveEvidenceInRanges(mediaEvidence, focusedInput, plan, durationSecs, watched.ranges),
|
|
208
|
+
retrieve(mediaEvidence, focusedInput, plan, durationSecs),
|
|
209
|
+
]);
|
|
210
|
+
hits = mergeHits(freshWatchedEvidence, retrieved);
|
|
211
|
+
assessment = assessEvidence(hits, focusedInput.query, plan, durationSecs, investigation, watched);
|
|
212
|
+
return assessment.sufficient;
|
|
213
|
+
},
|
|
214
|
+
});
|
|
215
|
+
analyzedRanges = outcome.analyzed;
|
|
216
|
+
inspection = outcome.lastResult;
|
|
217
|
+
// Watching the source is how a claim gets confirmed, so a service
|
|
218
|
+
// that cannot run leaves the claim unconfirmed -- but it does not
|
|
219
|
+
// erase what the index already found. Fail the turn only when
|
|
220
|
+
// there is nothing to show; otherwise return the indexed evidence
|
|
221
|
+
// and say plainly that it could not be checked against the source.
|
|
222
|
+
if (outcome.failure) {
|
|
223
|
+
analysisUnavailable = outcome.failure;
|
|
224
|
+
if (hits.length === 0) {
|
|
225
|
+
return {
|
|
226
|
+
success: false,
|
|
227
|
+
mediaAssetId: input.mediaAssetId,
|
|
228
|
+
error: `${outcome.failure} This is an analysis-service issue, not missing video evidence; do not answer the factual question as unconfirmed.`,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
const evidence = evidenceHitsFor(hits, assessment, plan, focusedInput, durationSecs);
|
|
235
|
+
const supportingClip = plan.kinds.includes('outcome')
|
|
236
|
+
? evidence.at(-1)?.timeRange
|
|
237
|
+
: evidence[0]?.timeRange;
|
|
238
|
+
const responseTimeMs = Date.now() - startedAt;
|
|
239
|
+
const answerPath = analyzedRanges.length > 0 ? 'rag+analysis' : 'rag';
|
|
240
|
+
void trackUsageEvent({
|
|
241
|
+
type: 'media_processing',
|
|
242
|
+
mediaType: asset.type === 'audio' ? 'audio' : 'video',
|
|
243
|
+
mediaOperation: 'investigation',
|
|
244
|
+
mediaAssetId: input.mediaAssetId,
|
|
245
|
+
queryKind: plan.kinds.join(','),
|
|
246
|
+
cache: answerPath === 'rag' ? 'hit' : 'miss',
|
|
247
|
+
evidenceCount: evidence.length,
|
|
248
|
+
frameCount: analyzedRanges.length,
|
|
249
|
+
durationSecs: analyzedRanges.reduce((total, range) => total + range.endSecs - range.startSecs, 0),
|
|
250
|
+
latencyMs: responseTimeMs,
|
|
251
|
+
timestamp: new Date().toISOString(),
|
|
252
|
+
});
|
|
253
|
+
return {
|
|
254
|
+
success: true,
|
|
255
|
+
mediaAssetId: input.mediaAssetId,
|
|
256
|
+
fileName: asset.fileName,
|
|
257
|
+
evidence,
|
|
258
|
+
...(supportingClip ? { supportingClip } : {}),
|
|
259
|
+
...(assessment.sufficient && supportingClip
|
|
260
|
+
? {
|
|
261
|
+
// The UI contract is generic and tool-owned. The host only
|
|
262
|
+
// renders a citations surface; it does not know why this
|
|
263
|
+
// source range was selected or expose extraction details.
|
|
264
|
+
ui: citationSurface(asset, context, evidence, supportingClip.startSecs),
|
|
265
|
+
}
|
|
266
|
+
: {}),
|
|
267
|
+
claimVerification: {
|
|
268
|
+
status: assessment.establishedByTrail
|
|
269
|
+
? 'established-by-trail'
|
|
270
|
+
: assessment.sufficient
|
|
271
|
+
? 'directly-established'
|
|
272
|
+
: 'needs-corroboration',
|
|
273
|
+
directlyEstablished: assessment.sufficient && !assessment.establishedByTrail,
|
|
274
|
+
rule: answeringRule(assessment, plan, analysisUnavailable),
|
|
275
|
+
},
|
|
276
|
+
investigation: {
|
|
277
|
+
answerPath,
|
|
278
|
+
responseTimeMs,
|
|
279
|
+
analyzedRanges,
|
|
280
|
+
broadCoverage: plan.requiresBroadCoverage === true,
|
|
281
|
+
coverage: investigation?.coverage,
|
|
282
|
+
},
|
|
283
|
+
...(focusedInput.exhaustive
|
|
284
|
+
? { continuation: exhaustiveContinuation(hits, assessment, focusedInput) }
|
|
285
|
+
: {}),
|
|
286
|
+
...(inspection ? { inspection } : {}),
|
|
287
|
+
...temporalContextResult(hits, plan),
|
|
288
|
+
};
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* What one re-watched window has to establish.
|
|
294
|
+
*
|
|
295
|
+
* The reader receives the user's whole question, which for a question spanning
|
|
296
|
+
* several moments describes the answer as a whole rather than the part this
|
|
297
|
+
* window can settle. Saying what this window is for is the difference between
|
|
298
|
+
* a reader summarising the passage and one reading the label off a shirt.
|
|
299
|
+
* Phrased from the question's shape, so it carries no assumption about subject
|
|
300
|
+
* matter.
|
|
301
|
+
*/
|
|
302
|
+
function windowObjective(plan) {
|
|
303
|
+
if (plan.subjectName) {
|
|
304
|
+
return `Which visible person is "${plan.subjectName}", and what the question asks about them.`;
|
|
305
|
+
}
|
|
306
|
+
if (plan.kinds.includes('state-change') || plan.kinds.includes('outcome')) {
|
|
307
|
+
return ('What changes during this window, what it changed from and to, and who or what brought it ' +
|
|
308
|
+
'about. Name whoever is responsible if anything on screen or in the speech names them -- a ' +
|
|
309
|
+
'caption, a label, a number worn or displayed, an announcement -- and say which of those ' +
|
|
310
|
+
'established the name. If nothing names them, say so instead of guessing.');
|
|
311
|
+
}
|
|
312
|
+
if (plan.kinds.includes('counting')) {
|
|
313
|
+
return 'How many of the thing being asked about are visible here, and how you counted them.';
|
|
314
|
+
}
|
|
315
|
+
if (plan.kinds.includes('evaluation')) {
|
|
316
|
+
return 'For each identified person, record only observable contributions or participation in this window, with what identifies the person. Do not rank them from this window alone.';
|
|
317
|
+
}
|
|
318
|
+
if (plan.kinds.includes('person-attribute') || plan.requiresIdentityContext) {
|
|
319
|
+
return 'Who is visible here, what identifies each of them, and the attribute being asked about.';
|
|
320
|
+
}
|
|
321
|
+
return undefined;
|
|
322
|
+
}
|
|
323
|
+
/** The evidence a reply cites, in the shape the host renders. */
|
|
324
|
+
function evidenceHitsFor(hits, assessment, plan, input, durationSecs) {
|
|
325
|
+
return selectAnswerEvidence(hits, assessment, plan, input, durationSecs).map((hit) => toEvidence(hit, plan));
|
|
326
|
+
}
|
|
327
|
+
async function retrieve(mediaEvidence, input, plan, durationSecs) {
|
|
328
|
+
const options = { queryPlan: plan, videoDurationSecs: durationSecs };
|
|
329
|
+
// A named person and an identity anchor are extra retrieval handles for the
|
|
330
|
+
// same index, not separate strategies. Issuing them alongside the ranked
|
|
331
|
+
// lookup costs one round trip instead of three.
|
|
332
|
+
const crossEvidence = isCrossEvidenceClaim(plan);
|
|
333
|
+
const [ranked, subjectHits, identityHits, visualHits, computedHits] = await Promise.all([
|
|
334
|
+
mediaEvidence.search(input.mediaAssetId, input.query, 16, options),
|
|
335
|
+
plan.subjectName
|
|
336
|
+
? mediaEvidence.search(input.mediaAssetId, plan.subjectName, 12, options)
|
|
337
|
+
: Promise.resolve([]),
|
|
338
|
+
plan.requiresIdentityContext
|
|
339
|
+
? mediaEvidence.search(input.mediaAssetId, 'participant names roster lineup name labels captions speaker identities', 8, options)
|
|
340
|
+
: Promise.resolve([]),
|
|
341
|
+
crossEvidence
|
|
342
|
+
? mediaEvidence.search(input.mediaAssetId, input.query, 16, {
|
|
343
|
+
...options,
|
|
344
|
+
modalities: ['visual'],
|
|
345
|
+
})
|
|
346
|
+
: Promise.resolve([]),
|
|
347
|
+
crossEvidence
|
|
348
|
+
? mediaEvidence.search(input.mediaAssetId, input.query, 16, {
|
|
349
|
+
...options,
|
|
350
|
+
modalities: ['computed'],
|
|
351
|
+
})
|
|
352
|
+
: Promise.resolve([]),
|
|
353
|
+
]);
|
|
354
|
+
if (!needsSourceWideView(plan)) {
|
|
355
|
+
return mergeHits(ranked, visualHits, computedHits, subjectHits, identityHits);
|
|
356
|
+
}
|
|
357
|
+
// A question about breadth or order cannot be served by the top matches
|
|
358
|
+
// alone: the relevant moment may share no words with the question. Scanning
|
|
359
|
+
// the immutable observations stays local and avoids a blind analysis pass.
|
|
360
|
+
const transitionTimeline = plan.kinds.includes('outcome') || plan.kinds.includes('state-change');
|
|
361
|
+
const timeline = await mediaEvidence.search(input.mediaAssetId, '', input.exhaustive ? 2_000 : 500, {
|
|
362
|
+
...options,
|
|
363
|
+
// A long transcript can contain more than 500 distinct moments before
|
|
364
|
+
// the source reaches its conclusion. For transition questions, read the
|
|
365
|
+
// compact reconciled accounts first so the final state cannot be cut off
|
|
366
|
+
// by hundreds of earlier speech/OCR records. Older indexes with no
|
|
367
|
+
// computed account fall back to the ordinary all-modality pass below.
|
|
368
|
+
...(transitionTimeline ? { modalities: ['computed'] } : {}),
|
|
369
|
+
// Keep independent modalities at one moment, but diversify repeated
|
|
370
|
+
// readings inside each modality. Re-inspecting a range can otherwise fill
|
|
371
|
+
// the whole timeline with near-identical records from that one minute.
|
|
372
|
+
minimumRangeDistanceSecs: 2,
|
|
373
|
+
});
|
|
374
|
+
const sourceTimeline = transitionTimeline && timeline.length < CORROBORATION_FLOOR
|
|
375
|
+
? await mediaEvidence.search(input.mediaAssetId, '', input.exhaustive ? 2_000 : 500, {
|
|
376
|
+
...options,
|
|
377
|
+
minimumRangeDistanceSecs: 2,
|
|
378
|
+
})
|
|
379
|
+
: timeline;
|
|
380
|
+
return plan.requiresIdentityContext
|
|
381
|
+
? mergeHits(ranked.slice(0, 3), identityHits.slice(0, 3), visualHits, computedHits, ranked, subjectHits, sourceTimeline)
|
|
382
|
+
: mergeHits(ranked, visualHits, computedHits, subjectHits, sourceTimeline);
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* Fetch observations emitted by the immediately preceding live inspection.
|
|
386
|
+
*
|
|
387
|
+
* This deliberately uses the ordinary evidence API, scoped to the watched
|
|
388
|
+
* time range. It avoids coupling the tool to a host-specific job payload and
|
|
389
|
+
* makes a newly indexed direct observation visible before relevance/diversity
|
|
390
|
+
* ranking can favour an older whole-video summary.
|
|
391
|
+
*/
|
|
392
|
+
async function retrieveEvidenceInRanges(mediaEvidence, input, plan, durationSecs, ranges) {
|
|
393
|
+
if (ranges.length === 0)
|
|
394
|
+
return [];
|
|
395
|
+
const options = {
|
|
396
|
+
queryPlan: plan,
|
|
397
|
+
videoDurationSecs: durationSecs,
|
|
398
|
+
modalities: ['visual', 'transcript', 'ocr', 'computed'],
|
|
399
|
+
// Keep all fresh observations from this compact range. The agent uses
|
|
400
|
+
// provenance and claim verification below to decide which can answer.
|
|
401
|
+
minimumRangeDistanceSecs: 0,
|
|
402
|
+
};
|
|
403
|
+
const groups = await Promise.all(ranges.map((range) => mediaEvidence.search(input.mediaAssetId, input.query, 24, {
|
|
404
|
+
...options,
|
|
405
|
+
timeRange: range,
|
|
406
|
+
})));
|
|
407
|
+
return mergeHits(...groups);
|
|
408
|
+
}
|
|
409
|
+
function needsSourceWideView(plan) {
|
|
410
|
+
return (plan.requiresBroadCoverage === true ||
|
|
411
|
+
plan.kinds.includes('coverage') ||
|
|
412
|
+
plan.kinds.includes('state-change') ||
|
|
413
|
+
plan.kinds.includes('outcome'));
|
|
414
|
+
}
|
|
415
|
+
function assessEvidence(hits, question, plan, durationSecs, investigation, watched = { ranges: [], since: '' }) {
|
|
416
|
+
const needsBreadth = plan.requiresBroadCoverage === true ||
|
|
417
|
+
plan.kinds.includes('coverage') ||
|
|
418
|
+
plan.kinds.includes('state-change');
|
|
419
|
+
const usable = hits.filter((hit) => !hit.conflict &&
|
|
420
|
+
['transcript', 'ocr', 'visual', 'computed'].includes(hit.evidence.modality));
|
|
421
|
+
if (needsBreadth) {
|
|
422
|
+
const answerBearing = plan.kinds.includes('question-inventory')
|
|
423
|
+
? questionInventoryHits(usable, question)
|
|
424
|
+
: plan.kinds.includes('source-inventory')
|
|
425
|
+
? sourceInventoryHits(usable, question)
|
|
426
|
+
: plan.kinds.includes('entity-inventory')
|
|
427
|
+
? identityInventoryHits(usable, question)
|
|
428
|
+
: plan.kinds.includes('evaluation')
|
|
429
|
+
? evaluationEvidenceHits(usable)
|
|
430
|
+
: plan.kinds.includes('state-change')
|
|
431
|
+
? temporalSequenceHits(usable)
|
|
432
|
+
: usable;
|
|
433
|
+
const isQuestionInventory = plan.kinds.includes('question-inventory');
|
|
434
|
+
const isSourceInventory = plan.kinds.includes('source-inventory');
|
|
435
|
+
const isEntityInventory = plan.kinds.includes('entity-inventory');
|
|
436
|
+
const explicitQuestionInventoryReady = isQuestionInventory &&
|
|
437
|
+
answerBearing.some((hit) => hit
|
|
438
|
+
.sourceQuestionProvenance === 'explicit');
|
|
439
|
+
const explicitSourceInventoryReady = isSourceInventory &&
|
|
440
|
+
answerBearing.some((hit) => /^Source item \((?:heading|slide-item|board-item|list-item),\s*(?:spoken|visible)\):/im.test(evidenceText(hit.evidence.payload)));
|
|
441
|
+
const coverageRatio = isQuestionInventory || isSourceInventory || isEntityInventory
|
|
442
|
+
? answerBearing.length > 0
|
|
443
|
+
? 1
|
|
444
|
+
: 0
|
|
445
|
+
: sourceCoverageRatio(answerBearing, durationSecs);
|
|
446
|
+
const hierarchy = investigation?.coverage;
|
|
447
|
+
const hierarchyReady = !hierarchy || hierarchy.totalChapters > 0 || hierarchy.totalScenes > 0;
|
|
448
|
+
// A broad RAG timeline is already the product of watching and indexing
|
|
449
|
+
// the source. Re-watching fixed windows on every ordered-change question
|
|
450
|
+
// made analysis the primary path, added minutes to chat, and could return
|
|
451
|
+
// a worse partial account than the index. Distinct timestamped moments
|
|
452
|
+
// spanning the source are sufficient; direct analysis remains the
|
|
453
|
+
// fallback when that coverage test fails or evidence conflicts.
|
|
454
|
+
const distinctMoments = new Set(answerBearing
|
|
455
|
+
.filter((hit) => !spansWholeSource(hit, durationSecs) && isAccountOfMoment(hit))
|
|
456
|
+
.map((hit) => Math.floor(hit.evidence.timeRange.startSecs / 15))).size;
|
|
457
|
+
const sequenceReady = !plan.kinds.includes('state-change') || distinctMoments >= 3;
|
|
458
|
+
const reconciledSequenceReady = plan.kinds.includes('state-change') &&
|
|
459
|
+
distinctMoments >= 3 &&
|
|
460
|
+
answerBearing.every((hit) => hit.evidence.source?.provider === 'video-intelligence-index');
|
|
461
|
+
return {
|
|
462
|
+
needsBreadth,
|
|
463
|
+
coverageRatio,
|
|
464
|
+
established: answerBearing,
|
|
465
|
+
corroborating: answerBearing,
|
|
466
|
+
sufficient: isQuestionInventory
|
|
467
|
+
? explicitQuestionInventoryReady
|
|
468
|
+
: isSourceInventory
|
|
469
|
+
? explicitSourceInventoryReady
|
|
470
|
+
: isEntityInventory
|
|
471
|
+
? answerBearing.length > 0
|
|
472
|
+
: plan.kinds.includes('evaluation')
|
|
473
|
+
? evaluationEvidenceReady(answerBearing)
|
|
474
|
+
: reconciledSequenceReady ||
|
|
475
|
+
(answerBearing.length >= 3 && coverageRatio >= 0.6 && hierarchyReady && sequenceReady),
|
|
476
|
+
};
|
|
477
|
+
}
|
|
478
|
+
const direct = answerEstablishedHits(hits, question, plan.kinds, watched, durationSecs);
|
|
479
|
+
const reconciled = direct.length === 0 ? indexedReconciledAnswerHits(hits, plan, durationSecs) : [];
|
|
480
|
+
const trail = direct.length === 0 && reconciled.length === 0 && isCrossEvidenceClaim(plan)
|
|
481
|
+
? indexedCrossEvidenceTrail(hits, durationSecs, plan)
|
|
482
|
+
: [];
|
|
483
|
+
const indexedAttributes = indexedUnboundSubjectAttributes(hits, question, plan);
|
|
484
|
+
const requestsKnownIdentity = identityAnchorNames(hits).some((name) => question.normalize('NFKC').toLocaleLowerCase().includes(name));
|
|
485
|
+
const namedAttributeRequest = requestsKnownIdentity && plan.kinds.includes('person-attribute');
|
|
486
|
+
const establishedByCrossEvidence = !namedAttributeRequest &&
|
|
487
|
+
!plan.kinds.includes('person-attribute') &&
|
|
488
|
+
trail.length >= CORROBORATION_FLOOR &&
|
|
489
|
+
trailHasRequiredIdentity(hits, trail, plan);
|
|
490
|
+
// A request to describe each visible subject can be answered accurately by
|
|
491
|
+
// a corroborated set of source descriptions even when no personal name was
|
|
492
|
+
// ever shown. That is different from a question about a named person: the
|
|
493
|
+
// latter still needs a name-to-person grounding before an attribute may be
|
|
494
|
+
// attached to them.
|
|
495
|
+
const describesUnboundSubjects = !plan.subjectName &&
|
|
496
|
+
!requestsKnownIdentity &&
|
|
497
|
+
plan.requiresIdentityContext === true &&
|
|
498
|
+
plan.kinds.includes('person-attribute') &&
|
|
499
|
+
indexedAttributes.length > 0 &&
|
|
500
|
+
attributeCoverageReady(indexedAttributes, hits);
|
|
501
|
+
const unboundSubjectEvidence = indexedAttributes;
|
|
502
|
+
const established = direct.length > 0
|
|
503
|
+
? direct
|
|
504
|
+
: reconciled.length > 0
|
|
505
|
+
? reconciled
|
|
506
|
+
: describesUnboundSubjects
|
|
507
|
+
? unboundSubjectEvidence
|
|
508
|
+
: establishedByCrossEvidence
|
|
509
|
+
? trail
|
|
510
|
+
: [];
|
|
511
|
+
return {
|
|
512
|
+
needsBreadth,
|
|
513
|
+
coverageRatio: 1,
|
|
514
|
+
established,
|
|
515
|
+
corroborating: mergeHits(unboundSubjectEvidence, usable.filter(isAccountOfMoment)),
|
|
516
|
+
// A reconciled, multi-modal trail is answer-level evidence once any
|
|
517
|
+
// required identity is grounded. Other trails remain targeting context,
|
|
518
|
+
// so an ambiguous sequence still falls through to bounded inspection.
|
|
519
|
+
sufficient: direct.length > 0 ||
|
|
520
|
+
reconciled.length > 0 ||
|
|
521
|
+
describesUnboundSubjects ||
|
|
522
|
+
establishedByCrossEvidence,
|
|
523
|
+
establishedByTrail: direct.length === 0 && reconciled.length === 0 && established.length > 0,
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
/** Keep compact records that actually bind identities, roles, or memberships. */
|
|
527
|
+
function identityInventoryHits(hits, question) {
|
|
528
|
+
const asksForPeople = /\b(?:people|persons?|participants?|contestants?|players?|members?|speakers?|presenters?|hosts?|guests?|attendees?|men|women)\b/i.test(question) ||
|
|
529
|
+
/(?:الأشخاص|الاشخاص|المشاركين|المتسابقين|اللاعبين|الأعضاء|الاعضاء|المتحدثين|المقدمين|الضيوف|الرجال|السيدات)/u.test(question);
|
|
530
|
+
const groupedRequest = /\b(?:each|every|both)\s+(?:team|group|side|department|organization|organisation|class|panel)\b|\bof\s+(?:each|every|both)\b/i.test(question) || /(?:كل|كلا)\s+(?:فريق|مجموعة|قسم|منظمة|فصل)/u.test(question);
|
|
531
|
+
const selected = chronological(hits).filter((hit) => {
|
|
532
|
+
const text = evidenceText(hit.evidence.payload);
|
|
533
|
+
if (/^(?:Reconciled|Indexed) participant:\s*[^—\n]{2,120}/im.test(text))
|
|
534
|
+
return true;
|
|
535
|
+
const present = [...text.matchAll(/^Present:\s*([^—\n]{2,120})\s*—\s*([^\n]+)/gim)];
|
|
536
|
+
if (!asksForPeople)
|
|
537
|
+
return present.length > 0;
|
|
538
|
+
return present.some((match) => {
|
|
539
|
+
const name = match[1]?.trim() ?? '';
|
|
540
|
+
const role = match[2] ?? '';
|
|
541
|
+
return HUMAN_ROLE_LANGUAGE.test(role) && !GENERIC_IDENTITY_LANGUAGE.test(name);
|
|
542
|
+
});
|
|
543
|
+
});
|
|
544
|
+
if (groupedRequest) {
|
|
545
|
+
const ranked = [...selected].sort((left, right) => groupedIdentityStrength(right) - groupedIdentityStrength(left));
|
|
546
|
+
if (ranked.length > 0 && groupedIdentityStrength(ranked[0]) >= 20)
|
|
547
|
+
return [ranked[0]];
|
|
548
|
+
}
|
|
549
|
+
const seen = new Set();
|
|
550
|
+
return selected
|
|
551
|
+
.filter((hit) => {
|
|
552
|
+
const text = evidenceText(hit.evidence.payload).trim();
|
|
553
|
+
const key = text.normalize('NFKC').toLocaleLowerCase();
|
|
554
|
+
if (!key || seen.has(key))
|
|
555
|
+
return false;
|
|
556
|
+
seen.add(key);
|
|
557
|
+
return true;
|
|
558
|
+
})
|
|
559
|
+
.slice(0, 48);
|
|
560
|
+
}
|
|
561
|
+
function groupedIdentityStrength(hit) {
|
|
562
|
+
const text = primaryAccount(evidenceText(hit.evidence.payload));
|
|
563
|
+
const namedPeople = [...text.matchAll(/^Present:\s*([^—\n]{2,120})\s*—\s*([^\n]+)/gim)].filter((match) => HUMAN_ROLE_LANGUAGE.test(match[2] ?? '') &&
|
|
564
|
+
!GENERIC_IDENTITY_LANGUAGE.test(match[1]?.trim() ?? '')).length;
|
|
565
|
+
let groupedBindings = 0;
|
|
566
|
+
const serializedBindings = text.match(/^Claim bindings:\s*(.+)$/im)?.[1];
|
|
567
|
+
if (serializedBindings) {
|
|
568
|
+
try {
|
|
569
|
+
const bindings = JSON.parse(serializedBindings);
|
|
570
|
+
groupedBindings = bindings.filter((binding) => typeof binding?.subject === 'string' && /\s(?:and|&|و)\s|،|,/iu.test(binding.subject)).length;
|
|
571
|
+
}
|
|
572
|
+
catch {
|
|
573
|
+
groupedBindings = 0;
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
return (namedPeople * 4 +
|
|
577
|
+
groupedBindings * 8 +
|
|
578
|
+
Number(hit.score ?? 0));
|
|
579
|
+
}
|
|
580
|
+
/**
|
|
581
|
+
* A judgement about contribution needs representative, identity-bearing
|
|
582
|
+
* moments across the source. Return a compact spread for synthesis rather
|
|
583
|
+
* than every transcript fragment or one semantically lucky scene.
|
|
584
|
+
*/
|
|
585
|
+
function evaluationEvidenceHits(hits) {
|
|
586
|
+
const semanticAccounts = chronological(hits).filter((hit) => {
|
|
587
|
+
if (!isAccountOfMoment(hit) || hasNegativeVerdict(hit))
|
|
588
|
+
return false;
|
|
589
|
+
const text = evidenceText(hit.evidence.payload);
|
|
590
|
+
return (/^(?:Reconciled|Indexed)\s+(?:participant|event|context|state):/im.test(text) ||
|
|
591
|
+
/^Present:\s*[^—\n]{2,120}\s*—/im.test(text));
|
|
592
|
+
});
|
|
593
|
+
// Semantic scene notes already fuse speech, action, identity and timing.
|
|
594
|
+
// Raw transcript is the fallback for audio-only sources; mixing hundreds of
|
|
595
|
+
// tiny ASR fragments into a healthy visual index hides the actual activity.
|
|
596
|
+
const useful = semanticAccounts.length >= 3
|
|
597
|
+
? semanticAccounts
|
|
598
|
+
: mergeHits(semanticAccounts, chronological(hits).filter((hit) => hit.evidence.modality === 'transcript' && isAccountOfMoment(hit)));
|
|
599
|
+
return evenlySpaced(useful, 36);
|
|
600
|
+
}
|
|
601
|
+
/** A comparative judgement needs repeated named human activity, not scenery or score displays. */
|
|
602
|
+
function evaluationEvidenceReady(hits) {
|
|
603
|
+
const momentsByPerson = new Map();
|
|
604
|
+
for (const hit of hits) {
|
|
605
|
+
const text = primaryAccount(evidenceText(hit.evidence.payload));
|
|
606
|
+
for (const match of text.matchAll(/^Present:\s*([^—\n]{2,120})\s*—\s*([^\n]+)/gim)) {
|
|
607
|
+
const name = match[1]?.trim() ?? '';
|
|
608
|
+
if (!HUMAN_ROLE_LANGUAGE.test(match[2] ?? '') || GENERIC_IDENTITY_LANGUAGE.test(name)) {
|
|
609
|
+
continue;
|
|
610
|
+
}
|
|
611
|
+
const key = name.normalize('NFKC').toLocaleLowerCase();
|
|
612
|
+
const moments = momentsByPerson.get(key) ?? new Set();
|
|
613
|
+
moments.add(Math.floor(hit.evidence.timeRange.startSecs / 15));
|
|
614
|
+
momentsByPerson.set(key, moments);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
return [...momentsByPerson.values()].filter((moments) => moments.size >= 2).length >= 2;
|
|
618
|
+
}
|
|
619
|
+
/** Source-wide visible units extracted during indexing, kept separate from narrative notes. */
|
|
620
|
+
function sourceInventoryHits(hits, question) {
|
|
621
|
+
const text = question.normalize('NFKC').toLocaleLowerCase();
|
|
622
|
+
const wantsSlides = /\bslides?\b|(?:الشرائح|السلايدز)/u.test(text);
|
|
623
|
+
const wantsBoard = /\b(?:boards?|whiteboards?)\b|(?:السبورة|اللوح)/u.test(text);
|
|
624
|
+
const records = chronological(hits).filter((hit) => {
|
|
625
|
+
const account = evidenceText(hit.evidence.payload);
|
|
626
|
+
const kind = account.match(/^Source item \((heading|slide-item|board-item|list-item),\s*(?:spoken|visible)\):/im)?.[1];
|
|
627
|
+
if (!kind)
|
|
628
|
+
return false;
|
|
629
|
+
if (wantsSlides)
|
|
630
|
+
return kind === 'heading' || kind === 'slide-item';
|
|
631
|
+
if (wantsBoard)
|
|
632
|
+
return kind === 'heading' || kind === 'board-item';
|
|
633
|
+
return true;
|
|
634
|
+
});
|
|
635
|
+
const seen = new Set();
|
|
636
|
+
return records.filter((hit) => {
|
|
637
|
+
const key = evidenceText(hit.evidence.payload).normalize('NFKC').toLocaleLowerCase().trim();
|
|
638
|
+
if (!key || seen.has(key))
|
|
639
|
+
return false;
|
|
640
|
+
seen.add(key);
|
|
641
|
+
return true;
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
/** Prefer the index's reconciled trajectory over hundreds of unrelated raw signals. */
|
|
645
|
+
function temporalSequenceHits(hits) {
|
|
646
|
+
const reconciled = chronological(hits).filter((hit) => {
|
|
647
|
+
if (hit.evidence.source?.provider !== 'video-intelligence-index')
|
|
648
|
+
return false;
|
|
649
|
+
const text = evidenceText(hit.evidence.payload).trim();
|
|
650
|
+
return (/^Reconciled state:/i.test(text) ||
|
|
651
|
+
(/^Reconciled event:/i.test(text) && TRANSITION_LANGUAGE.test(text)));
|
|
652
|
+
});
|
|
653
|
+
// Older indexes may not contain a synthesized trajectory. Preserve their
|
|
654
|
+
// broad raw account so a targeted inspection can still use it.
|
|
655
|
+
return reconciled.length >= 3 ? reconciled : hits;
|
|
656
|
+
}
|
|
657
|
+
/** Keep only source-authored prompts for an exhaustive question inventory. */
|
|
658
|
+
function questionInventoryHits(hits, query) {
|
|
659
|
+
const seen = new Set();
|
|
660
|
+
const candidates = chronological(hits).flatMap((hit) => {
|
|
661
|
+
// A later chat inspection can itself contain the user's question. It is
|
|
662
|
+
// useful for that answer, but it was never part of the recorded source.
|
|
663
|
+
if (hit.evidence.source?.provider === 'video-intelligence-vision')
|
|
664
|
+
return [];
|
|
665
|
+
return sourceQuestionRecordsFromHit(hit).flatMap((record, index) => {
|
|
666
|
+
const normalizedQuestion = normalizeInventoryText(record.question);
|
|
667
|
+
if (!normalizedQuestion ||
|
|
668
|
+
/what the person reading (?:these|the) notes cares about|what the (?:viewer|reader) cares about/i.test(normalizedQuestion)) {
|
|
669
|
+
return [];
|
|
670
|
+
}
|
|
671
|
+
const key = [
|
|
672
|
+
normalizedQuestion,
|
|
673
|
+
normalizeInventoryText(record.context ?? ''),
|
|
674
|
+
normalizeInventoryText(record.answer ?? ''),
|
|
675
|
+
].join('|');
|
|
676
|
+
if (seen.has(key))
|
|
677
|
+
return [];
|
|
678
|
+
seen.add(key);
|
|
679
|
+
const lines = [`Question: ${record.question}`];
|
|
680
|
+
if (record.context)
|
|
681
|
+
lines.push(`Context: ${record.context}`);
|
|
682
|
+
if (record.answer)
|
|
683
|
+
lines.push(`Answer: ${record.answer}`);
|
|
684
|
+
return [
|
|
685
|
+
{
|
|
686
|
+
...hit,
|
|
687
|
+
sourceQuestionProvenance: record.explicit ? 'explicit' : 'legacy',
|
|
688
|
+
sourceQuestionChannel: record.channel,
|
|
689
|
+
sourceQuestionText: record.question,
|
|
690
|
+
evidence: {
|
|
691
|
+
...hit.evidence,
|
|
692
|
+
id: `${hit.evidence.id}:source-question:${index}`,
|
|
693
|
+
payload: { text: lines.join('\n') },
|
|
694
|
+
},
|
|
695
|
+
},
|
|
696
|
+
];
|
|
697
|
+
});
|
|
698
|
+
});
|
|
699
|
+
const asksForFormalPromptSection = /\b(?:competition|contest|quiz|game|exam|test|challenge|round)\b|(?:مسابقة|المسابقة|اختبار|تحدي|جولة|لعبة)/iu.test(query);
|
|
700
|
+
const visibleMoments = candidates
|
|
701
|
+
.filter((candidate) => candidate
|
|
702
|
+
.sourceQuestionChannel === 'visible')
|
|
703
|
+
.map((candidate) => candidate.evidence.timeRange.startSecs);
|
|
704
|
+
const sectionCandidates = asksForFormalPromptSection && visibleMoments.length > 0
|
|
705
|
+
? candidates.filter((candidate) => {
|
|
706
|
+
const record = candidate;
|
|
707
|
+
if (record.sourceQuestionChannel === 'visible')
|
|
708
|
+
return true;
|
|
709
|
+
const at = candidate.evidence.timeRange.startSecs;
|
|
710
|
+
if (at < Math.min(...visibleMoments) - 60 || at > Math.max(...visibleMoments) + 60)
|
|
711
|
+
return false;
|
|
712
|
+
// A standalone prompt should not contain the first-person planning
|
|
713
|
+
// and turn-taking language typical of a reaction or answer fragment.
|
|
714
|
+
const sourceText = record.sourceQuestionText ?? '';
|
|
715
|
+
if (/\b(?:i|i'm|ive|i've|you know|let me|look|wait)\b|(?:\bانا\b|\bأنا\b|\bشوف\b|\bبينا\b|\bيلا\b|\bبعرف\b|\bعارف\b|\bهسه\b)/iu.test(sourceText))
|
|
716
|
+
return false;
|
|
717
|
+
const questionCues = sourceText
|
|
718
|
+
.toLocaleLowerCase()
|
|
719
|
+
.match(/[\p{Letter}\p{Number}][\p{Letter}\p{Mark}\p{Number}_-]*/gu)
|
|
720
|
+
?.filter((word) => /^(?:who|what|which|how|why|when|where|من|مين|ما|ماذا|كيف|كم|ايه|إيه|أي|اي)$/iu.test(word)) ?? [];
|
|
721
|
+
return !questionCues.some((cue, index) => questionCues.findIndex((other) => normalizeInventoryText(other) === normalizeInventoryText(cue)) !== index);
|
|
722
|
+
})
|
|
723
|
+
: candidates;
|
|
724
|
+
// OCR and ASR often capture the same prompt with slight spelling or
|
|
725
|
+
// transcription differences. Collapse nearby copies and retain the visible
|
|
726
|
+
// or fullest reading, while preserving genuine repeats later in the source.
|
|
727
|
+
return sectionCandidates.filter((candidate) => {
|
|
728
|
+
const candidateRecord = candidate;
|
|
729
|
+
if (!candidateRecord.sourceQuestionText)
|
|
730
|
+
return true;
|
|
731
|
+
const nearbyCopies = sectionCandidates.filter((other) => {
|
|
732
|
+
const otherText = other
|
|
733
|
+
.sourceQuestionText;
|
|
734
|
+
return (Boolean(otherText) &&
|
|
735
|
+
Math.abs(other.evidence.timeRange.startSecs - candidate.evidence.timeRange.startSecs) <=
|
|
736
|
+
90 &&
|
|
737
|
+
inventoryQuestionsMatch(candidateRecord.sourceQuestionText, otherText));
|
|
738
|
+
});
|
|
739
|
+
if (nearbyCopies.length < 2)
|
|
740
|
+
return true;
|
|
741
|
+
const preferred = nearbyCopies.sort((left, right) => {
|
|
742
|
+
const leftRecord = left;
|
|
743
|
+
const rightRecord = right;
|
|
744
|
+
const channelLead = Number(rightRecord.sourceQuestionChannel === 'visible') -
|
|
745
|
+
Number(leftRecord.sourceQuestionChannel === 'visible');
|
|
746
|
+
if (channelLead !== 0)
|
|
747
|
+
return channelLead;
|
|
748
|
+
return ((rightRecord.sourceQuestionText?.length ?? 0) - (leftRecord.sourceQuestionText?.length ?? 0));
|
|
749
|
+
})[0];
|
|
750
|
+
return preferred === candidate;
|
|
751
|
+
});
|
|
752
|
+
}
|
|
753
|
+
function normalizeInventoryText(value) {
|
|
754
|
+
return value.normalize('NFKC').toLocaleLowerCase().replace(/\s+/g, ' ').trim();
|
|
755
|
+
}
|
|
756
|
+
function inventoryQuestionTokens(value) {
|
|
757
|
+
const normalized = value
|
|
758
|
+
.normalize('NFKD')
|
|
759
|
+
.replace(/[\u064b-\u065f\u0670]/gu, '')
|
|
760
|
+
.replace(/[أإآٱ]/gu, 'ا')
|
|
761
|
+
.replace(/ى/gu, 'ي')
|
|
762
|
+
.toLocaleLowerCase();
|
|
763
|
+
const words = normalized.match(/[\p{Letter}\p{Number}][\p{Letter}\p{Mark}\p{Number}_-]*/gu) ?? [];
|
|
764
|
+
const firstCue = words.findIndex((word) => /^(?:who|what|when|where|why|how|which|whose|whom|is|are|was|were|do|does|did|can|could|would|will|name|list|identify|describe|tell|give|من|ما|ماذا|متي|اين|كيف|كم|هل|اي|لماذا|مين|ايه|فين|امتي|ازاي|اذكر|حدد|سم|سمي)$/iu.test(word));
|
|
765
|
+
return new Set(firstCue > 0 ? words.slice(firstCue) : words);
|
|
766
|
+
}
|
|
767
|
+
function inventoryQuestionsMatch(left, right) {
|
|
768
|
+
const leftTokens = inventoryQuestionTokens(left);
|
|
769
|
+
const rightTokens = inventoryQuestionTokens(right);
|
|
770
|
+
if (leftTokens.size === 0 || rightTokens.size === 0)
|
|
771
|
+
return false;
|
|
772
|
+
const overlap = [...leftTokens].filter((token) => rightTokens.has(token)).length;
|
|
773
|
+
return overlap / Math.min(leftTokens.size, rightTokens.size) >= 0.6;
|
|
774
|
+
}
|
|
775
|
+
function isWellFormedSourceQuestion(record) {
|
|
776
|
+
const question = record.question.normalize('NFKC').trim();
|
|
777
|
+
if (question.length < 4)
|
|
778
|
+
return false;
|
|
779
|
+
if (record.channel === 'visible')
|
|
780
|
+
return true;
|
|
781
|
+
const punctuated = /[?؟]\s*$/u.test(question);
|
|
782
|
+
// ASR commonly drops question marks. Require a source-language question or
|
|
783
|
+
// request cue near the start; an answer/reaction that happens to contain a
|
|
784
|
+
// question word much later must not become a source inventory item.
|
|
785
|
+
const words = question
|
|
786
|
+
.toLocaleLowerCase()
|
|
787
|
+
.match(/[\p{Letter}\p{Number}][\p{Letter}\p{Mark}\p{Number}_-]*/gu) ?? [];
|
|
788
|
+
const cue = /^(?:who|what|when|where|why|how|which|whose|whom|is|are|was|were|do|does|did|can|could|would|will|name|list|identify|describe|tell|give|من|ما|ماذا|متى|أين|اين|كيف|كم|هل|أي|اي|لماذا|مين|إيه|ايه|فين|امتى|ازاي|اذكر|أذكر|حدد|سم|سمي)$/iu;
|
|
789
|
+
const cueIndex = words.slice(0, 4).findIndex((word) => cue.test(word));
|
|
790
|
+
if (cueIndex < 0)
|
|
791
|
+
return punctuated;
|
|
792
|
+
const meaningful = new Set(words.slice(cueIndex + 1));
|
|
793
|
+
const deictic = /^(?:this|that|it|one|thing|exactly|first|second|third|دي|ده|دا|هذه|هذا|هي|هو|بالظبط|بالضبط|الاولى|الأولى|الثانية|الثالثة)$/iu;
|
|
794
|
+
if (meaningful.size === 1 && [...meaningful].every((word) => deictic.test(word)))
|
|
795
|
+
return false;
|
|
796
|
+
return true;
|
|
797
|
+
}
|
|
798
|
+
function sourceQuestionRecordsFromHit(hit) {
|
|
799
|
+
const text = evidenceText(hit.evidence.payload);
|
|
800
|
+
if (/^Question:\s*[^\n]+/i.test(text)) {
|
|
801
|
+
const question = text.match(/^Question:\s*([^\n]+)/i)?.[1]?.trim();
|
|
802
|
+
return question ? [{ question, explicit: true }] : [];
|
|
803
|
+
}
|
|
804
|
+
const context = inventoryContext(text);
|
|
805
|
+
const explicit = [];
|
|
806
|
+
const lines = text.split(/\r?\n/);
|
|
807
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
808
|
+
const sourceMatch = lines[index]?.match(/^Source question \((spoken|visible)\):\s*(.+)$/i);
|
|
809
|
+
const question = sourceMatch?.[2]?.trim();
|
|
810
|
+
if (!question)
|
|
811
|
+
continue;
|
|
812
|
+
const answer = lines[index + 1]?.match(/^Source answer:\s*(.+)$/i)?.[1]?.trim();
|
|
813
|
+
explicit.push({
|
|
814
|
+
question,
|
|
815
|
+
answer,
|
|
816
|
+
context,
|
|
817
|
+
explicit: true,
|
|
818
|
+
channel: sourceMatch?.[1]?.toLocaleLowerCase(),
|
|
819
|
+
});
|
|
820
|
+
}
|
|
821
|
+
if (explicit.length > 0)
|
|
822
|
+
return explicit.filter(isWellFormedSourceQuestion);
|
|
823
|
+
// Compatibility for indexes produced before sourceQuestions existed. Only
|
|
824
|
+
// accept text visibly presented as a question, or a claim question that is
|
|
825
|
+
// independently echoed by the observation. The analysis prompt by itself
|
|
826
|
+
// is deliberately not source evidence.
|
|
827
|
+
const visibleQuestions = [...text.matchAll(/^On screen:\s*['“"]?(.+?[?؟])['”"]?(?:\s+—.*)?$/gim)]
|
|
828
|
+
.map((match) => match[1]?.trim())
|
|
829
|
+
.filter((question) => Boolean(question));
|
|
830
|
+
if (visibleQuestions.length > 0)
|
|
831
|
+
return visibleQuestions.map((question) => ({ question, context, explicit: false }));
|
|
832
|
+
// `Claim question` is the analysis request supplied to the vision model,
|
|
833
|
+
// never source-authored content. Older observations sometimes paraphrased
|
|
834
|
+
// that request in their summary, which made a lexical echo look like a
|
|
835
|
+
// question from the recording. Only explicitly transcribed source questions
|
|
836
|
+
// or visibly quoted legacy questions are safe inventory entries.
|
|
837
|
+
return [];
|
|
838
|
+
}
|
|
839
|
+
function inventoryContext(text) {
|
|
840
|
+
const summary = text
|
|
841
|
+
.split(/(?:^|\n)(?:Present:|On screen:|Happened(?: \(inferred\))?:|Direct component:|Source question \(|Claim question:)/i, 1)[0]
|
|
842
|
+
?.replace(/^Observed context \(not a complete answer\):\s*/i, '')
|
|
843
|
+
.trim();
|
|
844
|
+
return summary ? summary.slice(0, 600) : undefined;
|
|
845
|
+
}
|
|
846
|
+
function temporalContextResult(hits, plan) {
|
|
847
|
+
if (!plan.kinds.includes('outcome') && !plan.kinds.includes('state-change'))
|
|
848
|
+
return {};
|
|
849
|
+
const readings = chronological(temporalSequenceHits(hits))
|
|
850
|
+
.filter((hit) => {
|
|
851
|
+
const text = evidenceText(hit.evidence.payload).trim();
|
|
852
|
+
return (hit.evidence.source?.provider === 'video-intelligence-index' &&
|
|
853
|
+
/^(?:Reconciled|Indexed)\s+(?:state|event):/i.test(text) &&
|
|
854
|
+
!hasNegativeVerdict(hit));
|
|
855
|
+
})
|
|
856
|
+
.filter((hit, index, all) => {
|
|
857
|
+
const key = `${Math.floor(hit.evidence.timeRange.startSecs)}:${evidenceText(hit.evidence.payload)}`;
|
|
858
|
+
return (all.findIndex((candidate) => `${Math.floor(candidate.evidence.timeRange.startSecs)}:${evidenceText(candidate.evidence.payload)}` === key) === index);
|
|
859
|
+
})
|
|
860
|
+
.slice(-48)
|
|
861
|
+
.map((hit) => ({
|
|
862
|
+
atSecs: hit.evidence.timeRange.startSecs,
|
|
863
|
+
text: evidenceText(hit.evidence.payload).slice(0, 700),
|
|
864
|
+
}));
|
|
865
|
+
if (readings.length === 0)
|
|
866
|
+
return {};
|
|
867
|
+
return {
|
|
868
|
+
temporalContext: {
|
|
869
|
+
readings,
|
|
870
|
+
rule: 'Read the timestamped states as a trajectory. Earlier and later counters can belong to ' +
|
|
871
|
+
'different scopes: a reset or a change in labels, unit, or scale is a scope boundary. ' +
|
|
872
|
+
'Resolve the ending state of the sequence relevant to the question from its changes and ' +
|
|
873
|
+
'labels. Within one uninterrupted scope, an unlabeled state inherits the nearest earlier ' +
|
|
874
|
+
'stable side labels; it does not inherit identities from an older scope. For an outcome, ' +
|
|
875
|
+
'compare the ending settled values and answer at the side/team/role granularity those ' +
|
|
876
|
+
'labels establish. Do not turn the leading side into a personal outcome unless this same ' +
|
|
877
|
+
'scope explicitly binds that person to the side. Do not substitute an earlier state merely ' +
|
|
878
|
+
'because it contains queried names, and do not rely on the last frame alone.',
|
|
879
|
+
},
|
|
880
|
+
};
|
|
881
|
+
}
|
|
882
|
+
/**
|
|
883
|
+
* The index synthesizer can reconcile several source readings into one
|
|
884
|
+
* timestamped account. One semantically matched account can settle an
|
|
885
|
+
* unbound state because it is already the result of indexing-time analysis.
|
|
886
|
+
* Identity attribution remains stricter: similarity locates the moment but
|
|
887
|
+
* never proves who the state belongs to.
|
|
888
|
+
*/
|
|
889
|
+
function indexedReconciledAnswerHits(hits, plan, durationSecs) {
|
|
890
|
+
// Semantic similarity is a locator, not an identity proof. A single
|
|
891
|
+
// reconciled record may settle an unbound state, but questions asking who
|
|
892
|
+
// require either a direct verdict or a trail joined to an identity anchor.
|
|
893
|
+
if (!isCrossEvidenceClaim(plan) || plan.requiresIdentityContext)
|
|
894
|
+
return [];
|
|
895
|
+
return hits.filter((hit) => {
|
|
896
|
+
const provider = hit.evidence.source?.provider;
|
|
897
|
+
const confidence = Number(hit.evidence.confidence?.score ?? 0);
|
|
898
|
+
const semantic = Number(hit.components?.semantic ?? 0);
|
|
899
|
+
const lexical = Number(hit.components?.lexical ?? 0);
|
|
900
|
+
const text = evidenceText(hit.evidence.payload).trim();
|
|
901
|
+
return (provider === 'video-intelligence-index' &&
|
|
902
|
+
hit.evidence.modality === 'computed' &&
|
|
903
|
+
confidence >= 0.5 &&
|
|
904
|
+
text.length >= 20 &&
|
|
905
|
+
(semantic >= 0.05 || lexical >= 0.2) &&
|
|
906
|
+
!hasNegativeVerdict(hit) &&
|
|
907
|
+
!LIMITATION_LANGUAGE.test(text) &&
|
|
908
|
+
!spansWholeSource(hit, durationSecs));
|
|
909
|
+
});
|
|
910
|
+
}
|
|
911
|
+
/**
|
|
912
|
+
* A visual observation already indexed for an attribute question is useful
|
|
913
|
+
* evidence even where the source never establishes a personal name. Keep the
|
|
914
|
+
* relation generic: it permits a source-backed description of visible people,
|
|
915
|
+
* but never attaches it to a guessed identity or claims a complete roster.
|
|
916
|
+
*/
|
|
917
|
+
function indexedUnboundSubjectAttributes(hits, question, plan) {
|
|
918
|
+
if (plan.subjectName ||
|
|
919
|
+
plan.requiresIdentityContext !== true ||
|
|
920
|
+
!plan.kinds.includes('person-attribute')) {
|
|
921
|
+
return [];
|
|
922
|
+
}
|
|
923
|
+
return hits.filter((hit) => (hit.evidence.modality === 'visual' ||
|
|
924
|
+
(hit.evidence.modality === 'computed' &&
|
|
925
|
+
hit.evidence.source?.provider === 'video-intelligence-index')) &&
|
|
926
|
+
isAccountOfMoment(hit) &&
|
|
927
|
+
// Similarity locates a scene, but a scene can be semantically close
|
|
928
|
+
// while describing a different attribute entirely. The account itself
|
|
929
|
+
// must contain the generic action/appearance relation requested here.
|
|
930
|
+
attributeEvidenceMatchesQuestion(primaryAccount(evidenceText(hit.evidence.payload)), question) &&
|
|
931
|
+
!hasNegativeVerdictForQuestion(hit, question));
|
|
932
|
+
}
|
|
933
|
+
function attributeEvidenceMatchesQuestion(evidence, question) {
|
|
934
|
+
if (!ATTRIBUTE_ACTION_LANGUAGE.test(evidence))
|
|
935
|
+
return false;
|
|
936
|
+
const requested = question.match(new RegExp(ATTRIBUTE_ACTION_LANGUAGE.source, 'giu')) ?? [];
|
|
937
|
+
if (requested.length === 0)
|
|
938
|
+
return true;
|
|
939
|
+
const source = evidence.normalize('NFKC').toLocaleLowerCase();
|
|
940
|
+
return requested.some((term) => {
|
|
941
|
+
const normalized = term.normalize('NFKC').toLocaleLowerCase();
|
|
942
|
+
const root = normalized.length >= 5 ? normalized.slice(0, 4) : normalized;
|
|
943
|
+
return root.length >= 3 && source.includes(root);
|
|
944
|
+
});
|
|
945
|
+
}
|
|
946
|
+
/** Do not call a two-person description complete when the index establishes a wider group. */
|
|
947
|
+
function attributeCoverageReady(attributes, allHits) {
|
|
948
|
+
const described = Math.max(0, ...attributes.map(explicitAttributedPeopleInAccount));
|
|
949
|
+
const known = Math.max(0, ...allHits.map(explicitPeopleInAccount));
|
|
950
|
+
return described > 0 && (known === 0 || described >= known);
|
|
951
|
+
}
|
|
952
|
+
function explicitAttributedPeopleInAccount(hit) {
|
|
953
|
+
const text = primaryAccount(evidenceText(hit.evidence.payload));
|
|
954
|
+
const clauses = text
|
|
955
|
+
.split(/[;؛.\n]+/u)
|
|
956
|
+
.map((clause) => clause.trim())
|
|
957
|
+
.filter((clause) => HUMAN_ROLE_LANGUAGE.test(clause) && ATTRIBUTE_ACTION_LANGUAGE.test(clause));
|
|
958
|
+
const positions = new Set(clauses.flatMap((clause) => [
|
|
959
|
+
...clause.matchAll(/\b(left|right|middle|center|centre|front|back)\s+(?:person|participant|contestant|player|member|speaker|presenter|host|guest|attendee|individual|man|woman)\b/gi),
|
|
960
|
+
].map((match) => match[0].toLocaleLowerCase()))).size;
|
|
961
|
+
const present = [...text.matchAll(/^Present:\s*([^—\n]{2,120})\s*—\s*([^\n]+)/gim)].filter((match) => ATTRIBUTE_ACTION_LANGUAGE.test(match[2] ?? '')).length;
|
|
962
|
+
const participantRecords = clauses.filter((clause) => /^(?:Reconciled|Indexed) participant:/i.test(clause)).length;
|
|
963
|
+
return Math.max(positions, present, participantRecords, clauses.length, 0);
|
|
964
|
+
}
|
|
965
|
+
function explicitPeopleInAccount(hit) {
|
|
966
|
+
const text = primaryAccount(evidenceText(hit.evidence.payload));
|
|
967
|
+
const present = [...text.matchAll(/^Present:\s*([^—\n]{2,120})\s*—\s*([^\n]+)/gim)].filter((match) => HUMAN_ROLE_LANGUAGE.test(match[2] ?? '')).length;
|
|
968
|
+
const numeric = [
|
|
969
|
+
...text.matchAll(/\b(\d{1,2})\s+(?:people|persons?|participants?|contestants?|players?|members?|speakers?|presenters?|hosts?|guests?|attendees?|men|women)\b/gi),
|
|
970
|
+
].map((match) => Number(match[1]));
|
|
971
|
+
const wordCounts = {
|
|
972
|
+
two: 2,
|
|
973
|
+
three: 3,
|
|
974
|
+
four: 4,
|
|
975
|
+
five: 5,
|
|
976
|
+
six: 6,
|
|
977
|
+
seven: 7,
|
|
978
|
+
eight: 8,
|
|
979
|
+
nine: 9,
|
|
980
|
+
ten: 10,
|
|
981
|
+
};
|
|
982
|
+
const words = [
|
|
983
|
+
...text.matchAll(/\b(two|three|four|five|six|seven|eight|nine|ten)\s+(?:people|persons?|participants?|contestants?|players?|members?|speakers?|presenters?|hosts?|guests?|attendees?|men|women)\b/gi),
|
|
984
|
+
].map((match) => wordCounts[match[1].toLocaleLowerCase()] ?? 0);
|
|
985
|
+
const positionalSubjects = new Set([
|
|
986
|
+
...text.matchAll(/\b(left|right|middle|center|centre|front|back)\s+(?:person|participant|contestant|player|member|speaker|presenter|host|guest|attendee|individual|man|woman)\b/gi),
|
|
987
|
+
].map((match) => match[0].toLocaleLowerCase())).size;
|
|
988
|
+
return Math.max(present, positionalSubjects, ...numeric, ...words, 0);
|
|
989
|
+
}
|
|
990
|
+
function primaryAccount(text) {
|
|
991
|
+
return text.split(/\nContinuity from (?:the )?(?:preceding|previous) moment:/i, 1)[0] ?? text;
|
|
992
|
+
}
|
|
993
|
+
/**
|
|
994
|
+
* Independent descriptions of the same bounded moment can establish a claim
|
|
995
|
+
* together even when no generated record repeats the user's wording. This is
|
|
996
|
+
* deliberately local in time: two generally relevant snippets from different
|
|
997
|
+
* parts of a long recording are not treated as corroboration.
|
|
998
|
+
*/
|
|
999
|
+
function indexedCrossEvidenceTrail(hits, durationSecs, plan) {
|
|
1000
|
+
const candidates = hits.filter((hit) => isAccountOfMoment(hit) && !hasNegativeVerdict(hit) && !spansWholeSource(hit, durationSecs));
|
|
1001
|
+
const transitionQuestion = plan.kinds.includes('outcome') || plan.kinds.includes('state-change');
|
|
1002
|
+
const seeds = [...candidates]
|
|
1003
|
+
.sort((left, right) => {
|
|
1004
|
+
const strength = (hit) => {
|
|
1005
|
+
const text = evidenceText(hit.evidence.payload);
|
|
1006
|
+
return (Number(hit.score ?? 0) +
|
|
1007
|
+
(transitionQuestion && TRANSITION_LANGUAGE.test(text) ? 0.25 : 0) -
|
|
1008
|
+
(LIMITATION_LANGUAGE.test(text) ? 0.35 : 0) +
|
|
1009
|
+
(transitionQuestion && durationSecs
|
|
1010
|
+
? Math.min(0.1, (hit.evidence.timeRange.endSecs / durationSecs) * 0.1)
|
|
1011
|
+
: 0));
|
|
1012
|
+
};
|
|
1013
|
+
return strength(right) - strength(left);
|
|
1014
|
+
})
|
|
1015
|
+
.slice(0, 32);
|
|
1016
|
+
// When the question asks how a state changed or concluded, the reconciled
|
|
1017
|
+
// state sequence is the evidence. A richly indexed but unrelated local
|
|
1018
|
+
// moment can otherwise win the clustering race and hide that trajectory.
|
|
1019
|
+
if (transitionQuestion) {
|
|
1020
|
+
const moments = new Set();
|
|
1021
|
+
const sequence = chronological(temporalSequenceHits(candidates).filter((hit) => hit.evidence.modality === 'computed' &&
|
|
1022
|
+
hit.evidence.source?.provider === 'video-intelligence-index' &&
|
|
1023
|
+
/^(?:Reconciled|Indexed)\s+(?:state|event):/i.test(evidenceText(hit.evidence.payload).trim()))).filter((hit) => {
|
|
1024
|
+
const moment = Math.floor(hit.evidence.timeRange.startSecs / 15);
|
|
1025
|
+
if (moments.has(moment))
|
|
1026
|
+
return false;
|
|
1027
|
+
moments.add(moment);
|
|
1028
|
+
return true;
|
|
1029
|
+
});
|
|
1030
|
+
if (sequence.length >= CORROBORATION_FLOOR)
|
|
1031
|
+
return sequence.slice(-12);
|
|
1032
|
+
}
|
|
1033
|
+
for (const seed of seeds) {
|
|
1034
|
+
const cluster = candidates.filter((hit) => hit.evidence.timeRange.startSecs <= seed.evidence.timeRange.endSecs + 5 &&
|
|
1035
|
+
hit.evidence.timeRange.endSecs >= seed.evidence.timeRange.startSecs - 5);
|
|
1036
|
+
const modalities = new Set(cluster.map((hit) => hit.evidence.modality));
|
|
1037
|
+
const hasReconciledReading = cluster.some((hit) => hit.evidence.source?.provider === 'video-intelligence-index');
|
|
1038
|
+
if (cluster.length >= CORROBORATION_FLOOR && modalities.size >= 2 && hasReconciledReading) {
|
|
1039
|
+
return reconciledFirst(cluster).slice(0, 12);
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
// A progression can also be established by the index's reconciled states
|
|
1043
|
+
// at separate moments. Keep this narrower than "several relevant hits":
|
|
1044
|
+
// only computed accounts produced by the index qualify, and identity-bound
|
|
1045
|
+
// questions still have to pass trailHasRequiredIdentity below.
|
|
1046
|
+
return [];
|
|
1047
|
+
}
|
|
1048
|
+
/**
|
|
1049
|
+
* How much of the source the evidence touches, measured in equal buckets.
|
|
1050
|
+
* The bucket count grows with length so a three-hour recording is not called
|
|
1051
|
+
* covered by three clips from its first minute.
|
|
1052
|
+
*/
|
|
1053
|
+
function sourceCoverageRatio(hits, durationSecs) {
|
|
1054
|
+
if (hits.length === 0)
|
|
1055
|
+
return 0;
|
|
1056
|
+
if (!durationSecs || durationSecs <= 0)
|
|
1057
|
+
return hits.length >= 3 ? 1 : 0;
|
|
1058
|
+
const buckets = Math.min(12, Math.max(3, Math.ceil(durationSecs / (10 * 60))));
|
|
1059
|
+
const covered = new Set(hits.map((hit) => Math.min(buckets - 1, Math.max(0, Math.floor((hit.evidence.timeRange.startSecs / durationSecs) * buckets)))));
|
|
1060
|
+
return covered.size / buckets;
|
|
1061
|
+
}
|
|
1062
|
+
/**
|
|
1063
|
+
* Where to look, chosen from what retrieval already found. Breadth questions
|
|
1064
|
+
* spread their looks across the source; specific questions look closely at
|
|
1065
|
+
* the best-matching moments. Nothing here knows what kind of video it is.
|
|
1066
|
+
*/
|
|
1067
|
+
function inspectionTargets(hits, question, plan, assessment, durationSecs, investigation, located = []) {
|
|
1068
|
+
// A conclusion needs the closing stretch *and* whatever retrieval ranked,
|
|
1069
|
+
// so it gets room for both rather than spending its whole budget on one.
|
|
1070
|
+
const budget = assessment.needsBreadth
|
|
1071
|
+
? 6
|
|
1072
|
+
: plan.requiresBothRanges || plan.kinds.includes('outcome')
|
|
1073
|
+
? 3
|
|
1074
|
+
: 2;
|
|
1075
|
+
const fromHits = rangesAroundHits(hits, durationSecs, budget * 2);
|
|
1076
|
+
const fromHierarchy = (investigation?.candidateRanges ?? []).map((range) => ({
|
|
1077
|
+
startSecs: Math.max(0, range.startSecs),
|
|
1078
|
+
endSecs: Math.min(durationSecs ?? range.endSecs, Math.min(range.endSecs, range.startSecs + MAX_INSPECTION_CHUNK_SECS)),
|
|
1079
|
+
}));
|
|
1080
|
+
// A question about how something concluded is answered where the source
|
|
1081
|
+
// concludes. Retrieval ranks by resemblance to the question, and the
|
|
1082
|
+
// closing moments often resemble it least -- they may show a result with
|
|
1083
|
+
// none of the question's words. Putting the ending first is what the word
|
|
1084
|
+
// "conclusion" means for any recording, not a fact about its subject.
|
|
1085
|
+
const closing = plan.kinds.includes('outcome') ? closingRanges(durationSecs) : [];
|
|
1086
|
+
// A question about how something changed is answered at the moments it
|
|
1087
|
+
// changed, and the index already recorded where those are. Aiming there beats
|
|
1088
|
+
// an even spread, which lands between the transitions and reports the states
|
|
1089
|
+
// either side of them without ever showing one happen.
|
|
1090
|
+
const transitions = plan.kinds.includes('state-change')
|
|
1091
|
+
? // An account of the whole progression needs every moment it moved.
|
|
1092
|
+
outcomeTransitionRanges(hits, durationSecs, { limit: budget, lateOnly: false })
|
|
1093
|
+
: plan.kinds.includes('outcome')
|
|
1094
|
+
? outcomeTransitionRanges(hits, durationSecs, { limit: 1 })
|
|
1095
|
+
: [];
|
|
1096
|
+
// For a conclusion, a located window inside the closing phase beats the
|
|
1097
|
+
// closing offsets, which only know how long the recording is. Elsewhere in
|
|
1098
|
+
// the source, located windows lead outright: they are the one signal that
|
|
1099
|
+
// reflects the question rather than the shape of the file.
|
|
1100
|
+
const phase = closingPhase(durationSecs);
|
|
1101
|
+
// A user-specified source position is stronger than semantic retrieval:
|
|
1102
|
+
// "at the beginning" should inspect the beginning even when a visually
|
|
1103
|
+
// similar later moment scores higher. These are generic timeline cues, not
|
|
1104
|
+
// assumptions about any particular video genre.
|
|
1105
|
+
const requestedPosition = temporalQuestionRanges(question, durationSecs);
|
|
1106
|
+
// Who is present is usually established early -- a roster, a title card, an
|
|
1107
|
+
// introduction -- so a question about identity is worth one look at the
|
|
1108
|
+
// opening. A question about how something concluded is not: its answer is at
|
|
1109
|
+
// the other end of the source, and spending a look on the opening there buys
|
|
1110
|
+
// a description of an entrance while the conclusion goes unread.
|
|
1111
|
+
const identityContext = plan.requiresIdentityContext && durationSecs && !plan.kinds.includes('outcome')
|
|
1112
|
+
? [{ startSecs: 0, endSecs: Math.min(MAX_INSPECTION_CHUNK_SECS, durationSecs) }]
|
|
1113
|
+
: [];
|
|
1114
|
+
const identityCoverage = plan.kinds.includes('person-attribute') ||
|
|
1115
|
+
plan.kinds.includes('entity-inventory') ||
|
|
1116
|
+
plan.kinds.includes('evaluation')
|
|
1117
|
+
? [...hits]
|
|
1118
|
+
.sort((left, right) => explicitPeopleInAccount(right) - explicitPeopleInAccount(left) ||
|
|
1119
|
+
left.evidence.timeRange.startSecs - right.evidence.timeRange.startSecs)
|
|
1120
|
+
.filter((hit) => explicitPeopleInAccount(hit) > 0)
|
|
1121
|
+
.slice(0, 1)
|
|
1122
|
+
.map((hit) => ({
|
|
1123
|
+
startSecs: Math.max(0, hit.evidence.timeRange.startSecs - TARGET_PADDING_SECS),
|
|
1124
|
+
endSecs: Math.min(durationSecs ?? hit.evidence.timeRange.endSecs + TARGET_PADDING_SECS, hit.evidence.timeRange.endSecs + TARGET_PADDING_SECS),
|
|
1125
|
+
}))
|
|
1126
|
+
: [];
|
|
1127
|
+
const locatedFirst = plan.kinds.includes('outcome') && phase
|
|
1128
|
+
? [
|
|
1129
|
+
...closing.slice(0, 1),
|
|
1130
|
+
...transitions,
|
|
1131
|
+
...closing.slice(1),
|
|
1132
|
+
...located.filter((range) => range.startSecs <= phase.endSecs && range.endSecs >= phase.startSecs),
|
|
1133
|
+
...located,
|
|
1134
|
+
]
|
|
1135
|
+
: [...transitions, ...located, ...closing];
|
|
1136
|
+
const candidates = uniqueRanges([
|
|
1137
|
+
...requestedPosition,
|
|
1138
|
+
...identityContext,
|
|
1139
|
+
...identityCoverage,
|
|
1140
|
+
...locatedFirst,
|
|
1141
|
+
...fromHits,
|
|
1142
|
+
...fromHierarchy,
|
|
1143
|
+
]);
|
|
1144
|
+
if (!assessment.needsBreadth) {
|
|
1145
|
+
// An index with nothing to say about the question is the strongest
|
|
1146
|
+
// reason to go and watch, not a reason to stop. With no moment to aim
|
|
1147
|
+
// at, sample the source instead of returning empty-handed.
|
|
1148
|
+
return (candidates.length > 0 ? candidates : evenlySpacedAnchors(durationSecs, budget)).slice(0, budget);
|
|
1149
|
+
}
|
|
1150
|
+
// Breadth needs the looks spread over the source rather than clustered on
|
|
1151
|
+
// whichever part retrieval happened to rank highest, so gaps in the index
|
|
1152
|
+
// get filled in by anchors of their own. Transitions are exempt from that
|
|
1153
|
+
// spreading: they are the specific moments the question is about, and
|
|
1154
|
+
// sampling evenly across the source would drop them for being close together.
|
|
1155
|
+
const spread = evenlySpaced(uniqueRanges([...candidates, ...evenlySpacedAnchors(durationSecs, budget)])
|
|
1156
|
+
.filter((range) => !transitions.some((moment) => moment.startSecs === range.startSecs))
|
|
1157
|
+
.sort((left, right) => left.startSecs - right.startSecs), Math.max(1, budget - transitions.length));
|
|
1158
|
+
return uniqueRanges([...transitions, ...spread]).slice(0, budget);
|
|
1159
|
+
}
|
|
1160
|
+
function temporalQuestionRanges(question, durationSecs) {
|
|
1161
|
+
if (!durationSecs || !Number.isFinite(durationSecs) || durationSecs <= 0)
|
|
1162
|
+
return [];
|
|
1163
|
+
const text = question.normalize('NFKC').toLocaleLowerCase();
|
|
1164
|
+
const opening = /\b(?:opening|beginning|start|initial)\b|(?:بداية|بدايه|في\s+الأول|في\s+الاول)/u.test(text);
|
|
1165
|
+
const ending = /\b(?:ending|closing|final)\b|(?:النهاية|النهايه|في\s+الآخر|في\s+الاخر)/u.test(text);
|
|
1166
|
+
const span = (side) => {
|
|
1167
|
+
const match = text.match(/(?:first|opening|beginning|initial|last|ending|closing|final)\s+(\d+(?:\.\d+)?)\s*(seconds?|secs?|minutes?|mins?)/u);
|
|
1168
|
+
const amount = match ? Number(match[1]) * (/min/i.test(match[2]) ? 60 : 1) : 60;
|
|
1169
|
+
const window = Math.min(durationSecs, Math.max(1, amount));
|
|
1170
|
+
return side === 'opening'
|
|
1171
|
+
? { startSecs: 0, endSecs: window }
|
|
1172
|
+
: { startSecs: Math.max(0, durationSecs - window), endSecs: durationSecs };
|
|
1173
|
+
};
|
|
1174
|
+
if (opening && !ending)
|
|
1175
|
+
return [span('opening')];
|
|
1176
|
+
if (ending && !opening)
|
|
1177
|
+
return [span('ending')];
|
|
1178
|
+
return [];
|
|
1179
|
+
}
|
|
1180
|
+
/**
|
|
1181
|
+
* An indexed account that explicitly describes a late transition is a better
|
|
1182
|
+
* first look than a fixed percentage of the file. It remains only a locator:
|
|
1183
|
+
* the source is still inspected before the claim is returned.
|
|
1184
|
+
*/
|
|
1185
|
+
function outcomeTransitionRanges(hits, durationSecs, options = {}) {
|
|
1186
|
+
const { limit = 2, lateOnly = true } = options;
|
|
1187
|
+
const accounts = hits.filter((hit) => {
|
|
1188
|
+
if (!isAccountOfMoment(hit) || hasNegativeVerdict(hit) || spansWholeSource(hit, durationSecs)) {
|
|
1189
|
+
return false;
|
|
1190
|
+
}
|
|
1191
|
+
const text = evidenceText(hit.evidence.payload);
|
|
1192
|
+
return TRANSITION_LANGUAGE.test(text) && !LIMITATION_LANGUAGE.test(text);
|
|
1193
|
+
});
|
|
1194
|
+
if (accounts.length === 0)
|
|
1195
|
+
return [];
|
|
1196
|
+
// A question about a conclusion looks late; a question about the whole
|
|
1197
|
+
// progression wants every moment it moved, wherever they fall.
|
|
1198
|
+
const lateAccounts = lateOnly && durationSecs
|
|
1199
|
+
? accounts.filter((hit) => {
|
|
1200
|
+
const start = hit.evidence.timeRange.startSecs / durationSecs;
|
|
1201
|
+
return start >= 0.6 && start <= 0.97;
|
|
1202
|
+
})
|
|
1203
|
+
: [];
|
|
1204
|
+
const ranked = (lateAccounts.length > 0 ? lateAccounts : accounts).sort((left, right) => {
|
|
1205
|
+
const retrievalDelta = Number(right.score ?? 0) -
|
|
1206
|
+
Number(left.score ?? 0);
|
|
1207
|
+
if (Math.abs(retrievalDelta) > 0.15)
|
|
1208
|
+
return retrievalDelta;
|
|
1209
|
+
return right.evidence.timeRange.startSecs - left.evidence.timeRange.startSecs;
|
|
1210
|
+
});
|
|
1211
|
+
return uniqueRanges(ranked.slice(0, limit).map((hit) => {
|
|
1212
|
+
// A retrieved transition usually identifies the lead-in; its resolving
|
|
1213
|
+
// frame or end-card often arrives immediately afterwards. End the
|
|
1214
|
+
// bounded look just beyond that evidence and keep the preceding
|
|
1215
|
+
// chronology, rather than centering it and cutting off the resolution.
|
|
1216
|
+
const endSecs = Math.min(durationSecs ?? hit.evidence.timeRange.endSecs + 9, hit.evidence.timeRange.endSecs + 9);
|
|
1217
|
+
const startSecs = Math.max(0, endSecs - OUTCOME_RESOLUTION_WINDOW_SECS);
|
|
1218
|
+
return {
|
|
1219
|
+
startSecs,
|
|
1220
|
+
endSecs,
|
|
1221
|
+
};
|
|
1222
|
+
}));
|
|
1223
|
+
}
|
|
1224
|
+
/**
|
|
1225
|
+
* Where a recording's content concludes. Not simply its last seconds: a
|
|
1226
|
+
* recording usually keeps running past the thing it recorded -- an outro,
|
|
1227
|
+
* credits, a sign-off, a trailer for something else -- so reading only the
|
|
1228
|
+
* final frames finds the tail rather than the conclusion. Three windows across
|
|
1229
|
+
* the closing stretch cover both, whatever the recording is of.
|
|
1230
|
+
*/
|
|
1231
|
+
function closingRanges(durationSecs) {
|
|
1232
|
+
if (!durationSecs || !Number.isFinite(durationSecs) || durationSecs <= 0)
|
|
1233
|
+
return [];
|
|
1234
|
+
const window = Math.min(MAX_INSPECTION_CHUNK_SECS, durationSecs);
|
|
1235
|
+
// Where the content ends comes first, the literal end second. A runtime that
|
|
1236
|
+
// grants one look at a time spends it on whichever range is offered first,
|
|
1237
|
+
// and the final seconds are an outro, a sign-off, or a trailer for the next
|
|
1238
|
+
// thing -- watching those and reporting back is how a concluded recording
|
|
1239
|
+
// gets described as not showing its conclusion.
|
|
1240
|
+
const ends = [
|
|
1241
|
+
Math.max(window, durationSecs * 0.9),
|
|
1242
|
+
Math.max(window, durationSecs * 0.95),
|
|
1243
|
+
durationSecs,
|
|
1244
|
+
];
|
|
1245
|
+
return uniqueRanges(ends.map((endSecs) => ({ startSecs: Math.max(0, endSecs - window), endSecs })));
|
|
1246
|
+
}
|
|
1247
|
+
function rangesAroundHits(hits, durationSecs, limit) {
|
|
1248
|
+
const ranges = [];
|
|
1249
|
+
for (const hit of hits) {
|
|
1250
|
+
const candidate = {
|
|
1251
|
+
startSecs: Math.max(0, hit.evidence.timeRange.startSecs - TARGET_PADDING_SECS),
|
|
1252
|
+
endSecs: Math.min(durationSecs ?? hit.evidence.timeRange.endSecs + TARGET_PADDING_SECS, hit.evidence.timeRange.endSecs + TARGET_PADDING_SECS),
|
|
1253
|
+
};
|
|
1254
|
+
if (candidate.endSecs <= candidate.startSecs)
|
|
1255
|
+
continue;
|
|
1256
|
+
if (ranges.some((range) => Math.abs(range.startSecs - candidate.startSecs) < 20))
|
|
1257
|
+
continue;
|
|
1258
|
+
ranges.push(candidate);
|
|
1259
|
+
if (ranges.length >= limit)
|
|
1260
|
+
break;
|
|
1261
|
+
}
|
|
1262
|
+
return ranges;
|
|
1263
|
+
}
|
|
1264
|
+
function evenlySpacedAnchors(durationSecs, count) {
|
|
1265
|
+
if (!durationSecs || durationSecs <= 0 || count <= 0)
|
|
1266
|
+
return [];
|
|
1267
|
+
const window = Math.min(MAX_INSPECTION_CHUNK_SECS, durationSecs / Math.max(count, 1));
|
|
1268
|
+
return Array.from({ length: count }, (_, index) => {
|
|
1269
|
+
const startSecs = Math.max(0, Math.min(durationSecs - window, (index * durationSecs) / count));
|
|
1270
|
+
return { startSecs, endSecs: Math.min(durationSecs, startSecs + window) };
|
|
1271
|
+
}).filter((range) => range.endSecs > range.startSecs);
|
|
1272
|
+
}
|
|
1273
|
+
/**
|
|
1274
|
+
* Runs the targeted looks. Independent ranges are dispatched together so a
|
|
1275
|
+
* chat turn waits once rather than once per range, and each wave re-checks
|
|
1276
|
+
* the index so a question answered by the first wave never pays for the rest.
|
|
1277
|
+
*
|
|
1278
|
+
* A runtime that admits one job at a time answers a concurrent request with a
|
|
1279
|
+
* busy signal instead of doing the work, so the first busy response drops the
|
|
1280
|
+
* remaining ranges back to one at a time rather than failing the turn.
|
|
1281
|
+
*/
|
|
1282
|
+
async function inspectRanges(options) {
|
|
1283
|
+
const { targets, input, plan, context, inspectVideoKnowledge, knownEntities, deadline, onWaveComplete, } = options;
|
|
1284
|
+
const analyzed = [];
|
|
1285
|
+
const pending = [...targets];
|
|
1286
|
+
let lastResult;
|
|
1287
|
+
let failure;
|
|
1288
|
+
// Targets are in priority order, and a runtime that allows one job at a
|
|
1289
|
+
// time turns the rest of a parallel wave away. Firing the whole wave anyway
|
|
1290
|
+
// means whichever range the runtime happened to accept is the one that gets
|
|
1291
|
+
// watched -- so a conclusion question could spend its entire budget on an
|
|
1292
|
+
// opening shot while the closing range was rejected and never retried.
|
|
1293
|
+
// Watch the best range first, on its own, and only widen once the runtime
|
|
1294
|
+
// has shown it can take more.
|
|
1295
|
+
let waveSize = 1;
|
|
1296
|
+
const request = (range) => inspectVideoKnowledge({
|
|
1297
|
+
mediaAssetId: input.mediaAssetId,
|
|
1298
|
+
startSecs: range.startSecs,
|
|
1299
|
+
endSecs: range.endSecs,
|
|
1300
|
+
purpose: inspectionPurpose(plan.kinds),
|
|
1301
|
+
queryId: stableQueryId(input.query),
|
|
1302
|
+
question: input.query,
|
|
1303
|
+
maxFrames: plan.kinds.includes('state-change')
|
|
1304
|
+
? 24
|
|
1305
|
+
: plan.kinds.includes('person-attribute') || plan.requiresIdentityContext
|
|
1306
|
+
? // Identity reads need chronology, but a very dense single VLM
|
|
1307
|
+
// request is less reliable than a compact, readable spread.
|
|
1308
|
+
8
|
|
1309
|
+
: plan.requiresBothRanges ||
|
|
1310
|
+
plan.kinds.includes('comparison') ||
|
|
1311
|
+
plan.kinds.includes('outcome')
|
|
1312
|
+
? 10
|
|
1313
|
+
: 14,
|
|
1314
|
+
...(knownEntities.length > 0 ? { knownEntities } : {}),
|
|
1315
|
+
// A close read costs more per range, so it is spent where a wrong
|
|
1316
|
+
// reading of one detail changes the answer, and the fast reader
|
|
1317
|
+
// covers the ranges that only need to say what is happening.
|
|
1318
|
+
analysisMode: needsCloseReading(plan) ? 'thorough' : 'fast',
|
|
1319
|
+
continuousSequence: true,
|
|
1320
|
+
includeSpeech: plan.kinds.includes('direct-speech') ||
|
|
1321
|
+
plan.kinds.includes('state-change') ||
|
|
1322
|
+
plan.kinds.includes('outcome') ||
|
|
1323
|
+
plan.requiresIdentityContext,
|
|
1324
|
+
// The client transport enforces this same deadline. Do not grant a
|
|
1325
|
+
// later range a fresh minimum wait: that turns a bounded chat turn
|
|
1326
|
+
// into several consecutive timeout windows when a worker is stalled.
|
|
1327
|
+
maxWaitMs: Math.max(1_000, deadline - Date.now()),
|
|
1328
|
+
}, context);
|
|
1329
|
+
while (pending.length > 0) {
|
|
1330
|
+
// The client transport owns the hard deadline. Do not start another
|
|
1331
|
+
// request once there is no meaningful time left for it to return.
|
|
1332
|
+
if (deadline - Date.now() < 1_000) {
|
|
1333
|
+
failure = 'Video analysis did not finish within the interactive response budget.';
|
|
1334
|
+
break;
|
|
1335
|
+
}
|
|
1336
|
+
const waveStartedAt = new Date().toISOString();
|
|
1337
|
+
const wave = pending.splice(0, waveSize);
|
|
1338
|
+
const results = await Promise.all(wave.map(request));
|
|
1339
|
+
const rejected = [];
|
|
1340
|
+
let busyMessage;
|
|
1341
|
+
for (const [position, result] of results.entries()) {
|
|
1342
|
+
if (isServiceBusy(result)) {
|
|
1343
|
+
rejected.push(wave[position]);
|
|
1344
|
+
busyMessage = isInspectionFailure(result) ? result.error : busyMessage;
|
|
1345
|
+
continue;
|
|
1346
|
+
}
|
|
1347
|
+
if (isInspectionFailure(result)) {
|
|
1348
|
+
failure = result.error;
|
|
1349
|
+
continue;
|
|
1350
|
+
}
|
|
1351
|
+
analyzed.push(wave[position]);
|
|
1352
|
+
lastResult = result;
|
|
1353
|
+
}
|
|
1354
|
+
if (rejected.length > 0) {
|
|
1355
|
+
// The runtime took one of these and turned the rest away. Put the
|
|
1356
|
+
// turned-away ranges back and stop asking for more than one at a time.
|
|
1357
|
+
if (waveSize === 1) {
|
|
1358
|
+
failure ??= busyMessage ?? 'Video analysis is busy with another request.';
|
|
1359
|
+
break;
|
|
1360
|
+
}
|
|
1361
|
+
waveSize = 1;
|
|
1362
|
+
pending.unshift(...rejected);
|
|
1363
|
+
failure = undefined;
|
|
1364
|
+
continue;
|
|
1365
|
+
}
|
|
1366
|
+
if (failure)
|
|
1367
|
+
break;
|
|
1368
|
+
if (await onWaveComplete({ ranges: analyzed, since: waveStartedAt }))
|
|
1369
|
+
break;
|
|
1370
|
+
// The runtime took everything it was offered, so it can take more at once.
|
|
1371
|
+
waveSize = Math.min(MAX_PARALLEL_INSPECTIONS, Math.max(waveSize * 2, 1), pending.length);
|
|
1372
|
+
}
|
|
1373
|
+
return { analyzed, lastResult, failure };
|
|
1374
|
+
}
|
|
1375
|
+
/**
|
|
1376
|
+
* Carries identities already established by the index into a bounded look.
|
|
1377
|
+
* This is especially important for comparisons involving several unnamed
|
|
1378
|
+
* subjects: the reader sees the pixels, while the index supplies the stable
|
|
1379
|
+
* names those pixels must be grounded against.
|
|
1380
|
+
*/
|
|
1381
|
+
function inspectionEntityHints(hits, plan) {
|
|
1382
|
+
const hints = [];
|
|
1383
|
+
if (plan.subjectName)
|
|
1384
|
+
hints.push(plan.subjectName);
|
|
1385
|
+
if (plan.requiresIdentityContext || plan.kinds.includes('comparison')) {
|
|
1386
|
+
for (const hit of hits) {
|
|
1387
|
+
const payload = hit.evidence.payload;
|
|
1388
|
+
if (payload && typeof payload === 'object' && !Array.isArray(payload)) {
|
|
1389
|
+
const subject = payload.subject;
|
|
1390
|
+
if (typeof subject === 'string' && subject.trim() && subject !== 'on-screen-text') {
|
|
1391
|
+
hints.push(subject.trim());
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
const text = evidenceText(payload);
|
|
1395
|
+
for (const match of text.matchAll(/(?:Reconciled|Indexed) participant:\s*([^—\n]{1,120})/gi)) {
|
|
1396
|
+
if (match[1]?.trim())
|
|
1397
|
+
hints.push(match[1].trim());
|
|
1398
|
+
}
|
|
1399
|
+
if (/^(?:Reconciled|Indexed)\s+(?:state|event|context|overview):/i.test(text)) {
|
|
1400
|
+
const account = text.replace(/^(?:Reconciled|Indexed)\s+(?:state|event|context|overview):\s*/i, '');
|
|
1401
|
+
// A reconciled display/state can establish names that are not people
|
|
1402
|
+
// and therefore do not belong in the participant list. Preserve
|
|
1403
|
+
// ordinary title-cased names as reader hints; they remain hints, not
|
|
1404
|
+
// identity claims, until the bounded source pass binds them.
|
|
1405
|
+
for (const match of account.matchAll(/\b[A-Z][\p{Letter}\p{Mark}'’.-]{2,}(?:\s+[A-Z][\p{Letter}\p{Mark}'’.-]{2,}){0,3}\b/gu)) {
|
|
1406
|
+
hints.push(match[0]);
|
|
1407
|
+
}
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
return [...new Map(hints.map((hint) => [hint.toLocaleLowerCase(), hint])).values()].slice(0, 20);
|
|
1412
|
+
}
|
|
1413
|
+
function needsCloseReading(plan) {
|
|
1414
|
+
return (plan.kinds.includes('exact-ocr') ||
|
|
1415
|
+
plan.kinds.includes('counting') ||
|
|
1416
|
+
// An unnamed, multi-person attribute request is a broad visual read: the
|
|
1417
|
+
// fast reader can describe its source-supported distinctions in one pass.
|
|
1418
|
+
// Reserve the slower precision model for linking an attribute to a
|
|
1419
|
+
// particular named person, where a mistaken association is materially
|
|
1420
|
+
// worse than a short follow-up inspection.
|
|
1421
|
+
(plan.kinds.includes('person-attribute') && Boolean(plan.subjectName)));
|
|
1422
|
+
}
|
|
1423
|
+
/**
|
|
1424
|
+
* The part of a recording where what it recorded actually concludes. A
|
|
1425
|
+
* recording runs on past that -- reactions, a sign-off, an end card -- so the
|
|
1426
|
+
* closing *phase* is the useful window; its final seconds are usually an outro
|
|
1427
|
+
* that says nothing about the question.
|
|
1428
|
+
*/
|
|
1429
|
+
/**
|
|
1430
|
+
* A record covering most of the recording -- a summary, an overview -- orients
|
|
1431
|
+
* the reader but locates nothing, so at most one belongs in a trail of
|
|
1432
|
+
* moments.
|
|
1433
|
+
*/
|
|
1434
|
+
function spansWholeSource(hit, durationSecs) {
|
|
1435
|
+
if (!durationSecs || durationSecs <= 0)
|
|
1436
|
+
return false;
|
|
1437
|
+
const span = hit.evidence.timeRange.endSecs - hit.evidence.timeRange.startSecs;
|
|
1438
|
+
return span >= durationSecs * 0.5;
|
|
1439
|
+
}
|
|
1440
|
+
function closingPhase(durationSecs) {
|
|
1441
|
+
if (!durationSecs || durationSecs <= 0)
|
|
1442
|
+
return undefined;
|
|
1443
|
+
return { startSecs: durationSecs * 0.66, endSecs: durationSecs };
|
|
1444
|
+
}
|
|
1445
|
+
function selectAnswerEvidence(hits, assessment, plan, input, durationSecs) {
|
|
1446
|
+
if (assessment.needsBreadth) {
|
|
1447
|
+
const limit = Math.min(input.limit ?? 36, 48);
|
|
1448
|
+
if (input.exhaustive) {
|
|
1449
|
+
const ordered = chronological(assessment.established);
|
|
1450
|
+
const cursor = Math.min(Math.max(0, input.cursor ?? 0), ordered.length);
|
|
1451
|
+
return ordered.slice(cursor, cursor + limit);
|
|
1452
|
+
}
|
|
1453
|
+
return chronological(assessment.established).length <= limit
|
|
1454
|
+
? chronological(assessment.established)
|
|
1455
|
+
: mergeHits(chronological(assessment.established).filter((hit) => /Claim verdict:\s*direct/i.test(evidenceText(hit.evidence.payload))), evenlySpaced(chronological(assessment.established), limit))
|
|
1456
|
+
.sort((left, right) => left.evidence.timeRange.startSecs - right.evidence.timeRange.startSecs)
|
|
1457
|
+
.slice(0, limit);
|
|
1458
|
+
}
|
|
1459
|
+
const limit = Math.min(input.limit ?? (plan.requiresBothRanges || isCrossEvidenceClaim(plan) ? 12 : 8), 12);
|
|
1460
|
+
if (assessment.establishedByTrail) {
|
|
1461
|
+
// A trail can exist while `established` is empty: the trail settles the
|
|
1462
|
+
// claim, but only a question about unnamed subjects promotes it into
|
|
1463
|
+
// `established`. Returning `established` alone then answered a perfectly
|
|
1464
|
+
// well-indexed question with no evidence at all -- the reply had nothing
|
|
1465
|
+
// to cite and said the source did not show it.
|
|
1466
|
+
const trail = assessment.established.length > 0 ? assessment.established : assessment.corroborating;
|
|
1467
|
+
return chronological(reconciledFirst(trail)).slice(0, limit);
|
|
1468
|
+
}
|
|
1469
|
+
if (!assessment.sufficient && isCrossEvidenceClaim(plan)) {
|
|
1470
|
+
const identity = plan.requiresIdentityContext ? hits.filter(isIdentityAnchor) : [];
|
|
1471
|
+
const stateChanges = hits
|
|
1472
|
+
.filter((hit) => {
|
|
1473
|
+
const text = evidenceText(hit.evidence.payload);
|
|
1474
|
+
return (TRANSITION_LANGUAGE.test(text) || /^(?:Reconciled|Indexed)\s+(?:state|event):/i.test(text));
|
|
1475
|
+
})
|
|
1476
|
+
.sort((left, right) => right.evidence.timeRange.startSecs - left.evidence.timeRange.startSecs);
|
|
1477
|
+
return chronological(mergeHits(identity, stateChanges, assessment.corroborating, hits).slice(0, limit));
|
|
1478
|
+
}
|
|
1479
|
+
const settled = plan.requiresInspectionWhenInsufficient
|
|
1480
|
+
? assessment.established.length > 0
|
|
1481
|
+
? assessment.established
|
|
1482
|
+
: hits
|
|
1483
|
+
: hits;
|
|
1484
|
+
// A direct, question-scoped observation is stronger than the surrounding
|
|
1485
|
+
// chronology. Do not dilute it with broad summaries or neighbouring signal
|
|
1486
|
+
// merely because the question also happens to be a comparison or outcome.
|
|
1487
|
+
// Those supporting records remain available when no direct answer exists.
|
|
1488
|
+
if (assessment.sufficient &&
|
|
1489
|
+
assessment.established.some((hit) => isDirectVerdict(hit, input.query))) {
|
|
1490
|
+
return reconciledFirst(assessment.established).slice(0, limit);
|
|
1491
|
+
}
|
|
1492
|
+
// A conclusion, a count, and a comparison are read across the source, so the
|
|
1493
|
+
// chronology is the answer material whether or not a bounded look settled
|
|
1494
|
+
// anything. Returning only what one 60-second look produced replaced that
|
|
1495
|
+
// chronology with a minute of raw on-screen text -- a successful look made
|
|
1496
|
+
// the answer worse. Records are chosen by relevance, then presented in time
|
|
1497
|
+
// order, because a trail shuffled by rank cannot be read as one.
|
|
1498
|
+
if (isCrossEvidenceClaim(plan) && assessment.corroborating.length >= CORROBORATION_FLOOR) {
|
|
1499
|
+
// A trail has to reach its end to be readable as one. Relevance alone
|
|
1500
|
+
// clusters near whatever the index describes most richly -- usually the
|
|
1501
|
+
// opening -- so the latest records are reserved a share of the slots
|
|
1502
|
+
// outright. Without this the chronology stopped a third of the way in and
|
|
1503
|
+
// the conclusion it was supposed to lead to was never in it.
|
|
1504
|
+
// Reserve the share for the *strongest* records from the closing phase
|
|
1505
|
+
// rather than the last ones in it. Taking the last ones fills the reserved
|
|
1506
|
+
// slots with the outro -- an end card and a sign-off -- and leaves out the
|
|
1507
|
+
// moment a few minutes earlier where the thing actually concluded.
|
|
1508
|
+
// Distinct moments only: one moment recorded four ways would consume the
|
|
1509
|
+
// whole share on its own.
|
|
1510
|
+
const phase = closingPhase(durationSecs);
|
|
1511
|
+
const closingCandidates = phase
|
|
1512
|
+
? // The index's own reconciled account of a moment states what was on
|
|
1513
|
+
// screen; a transcript line from the same moment is people talking
|
|
1514
|
+
// over it. A conclusion is read off the former.
|
|
1515
|
+
reconciledFirst(assessment.corroborating.filter((hit) => hit.evidence.timeRange.startSecs <= phase.endSecs &&
|
|
1516
|
+
hit.evidence.timeRange.endSecs >= phase.startSecs &&
|
|
1517
|
+
!spansWholeSource(hit, durationSecs)))
|
|
1518
|
+
: [];
|
|
1519
|
+
const latestMoments = [];
|
|
1520
|
+
// `corroborating` preserves retrieval's ranking, so taking these in order
|
|
1521
|
+
// is taking the best-ranked closing records.
|
|
1522
|
+
for (const hit of closingCandidates) {
|
|
1523
|
+
if (latestMoments.length >= Math.max(1, Math.floor(limit / 3)))
|
|
1524
|
+
break;
|
|
1525
|
+
if (latestMoments.some((kept) => Math.abs(kept.evidence.timeRange.startSecs - hit.evidence.timeRange.startSecs) < 15))
|
|
1526
|
+
continue;
|
|
1527
|
+
latestMoments.push(hit);
|
|
1528
|
+
}
|
|
1529
|
+
const latest = chronological(latestMoments);
|
|
1530
|
+
// One whole-source summary is orientation; four are four ways of saying
|
|
1531
|
+
// the same thing, and they were taking a third of the slots.
|
|
1532
|
+
let summaries = 0;
|
|
1533
|
+
// A moment that was watched several times, or read by several operators,
|
|
1534
|
+
// is recorded many ways. Two of them corroborate each other; seven of them
|
|
1535
|
+
// are one moment eating the whole trail, and the chronology stops being a
|
|
1536
|
+
// chronology.
|
|
1537
|
+
const perMoment = new Map();
|
|
1538
|
+
const trail = mergeHits(
|
|
1539
|
+
// Whatever actually settled the claim leads, but only the records that
|
|
1540
|
+
// recount a moment -- the raw signal alongside them corroborates and is
|
|
1541
|
+
// plentiful enough to crowd the trail out on its own.
|
|
1542
|
+
assessment.established.filter(isAccountOfMoment), latest, reconciledFirst(assessment.corroborating), reconciledFirst(settled)).filter((hit) => {
|
|
1543
|
+
if (spansWholeSource(hit, durationSecs))
|
|
1544
|
+
return ++summaries <= 1;
|
|
1545
|
+
const moment = Math.floor(hit.evidence.timeRange.startSecs / 15);
|
|
1546
|
+
const seen = (perMoment.get(moment) ?? 0) + 1;
|
|
1547
|
+
perMoment.set(moment, seen);
|
|
1548
|
+
return seen <= 2;
|
|
1549
|
+
});
|
|
1550
|
+
return chronological(trail.slice(0, limit));
|
|
1551
|
+
}
|
|
1552
|
+
// A question that contrasts two things is not answered by the moment that
|
|
1553
|
+
// settled it: each side needs its own evidence, and they are rarely in the
|
|
1554
|
+
// same moment. Carry the settling evidence plus a spread of the rest.
|
|
1555
|
+
if (plan.requiresBothRanges && settled.length < limit) {
|
|
1556
|
+
return chronological(mergeHits(settled, evenlySpaced(chronological(hits), limit - settled.length))).slice(0, limit);
|
|
1557
|
+
}
|
|
1558
|
+
return reconciledFirst(settled).slice(0, limit);
|
|
1559
|
+
}
|
|
1560
|
+
function isIdentityAnchor(hit) {
|
|
1561
|
+
const payload = hit.evidence.payload;
|
|
1562
|
+
if (payload && typeof payload === 'object' && !Array.isArray(payload)) {
|
|
1563
|
+
const subject = payload.subject;
|
|
1564
|
+
if (typeof subject === 'string' && subject.trim() && subject !== 'on-screen-text')
|
|
1565
|
+
return true;
|
|
1566
|
+
}
|
|
1567
|
+
return /(?:Reconciled|Indexed) participant:\s*[^—\n]{2,120}/i.test(evidenceText(payload));
|
|
1568
|
+
}
|
|
1569
|
+
function exhaustiveContinuation(hits, assessment, input) {
|
|
1570
|
+
const ordered = chronological(assessment.needsBreadth ? assessment.established : hits);
|
|
1571
|
+
const cursor = Math.min(Math.max(0, input.cursor ?? 0), ordered.length);
|
|
1572
|
+
const pageSize = Math.min(input.limit ?? 36, 48);
|
|
1573
|
+
const nextCursor = cursor + Math.min(pageSize, Math.max(0, ordered.length - cursor));
|
|
1574
|
+
const hasMore = nextCursor < ordered.length;
|
|
1575
|
+
return {
|
|
1576
|
+
exhaustive: true,
|
|
1577
|
+
totalItems: ordered.length,
|
|
1578
|
+
returnedItems: nextCursor - cursor,
|
|
1579
|
+
cursor,
|
|
1580
|
+
hasMore,
|
|
1581
|
+
...(hasMore ? { nextCursor } : {}),
|
|
1582
|
+
rule: hasMore
|
|
1583
|
+
? 'This is one chronological page, not the complete account. Call queryVideoEvidence again with exhaustive=true and cursor=nextCursor before finalizing the answer.'
|
|
1584
|
+
: 'Every chronological item in the active index has now been returned across the pages.',
|
|
1585
|
+
};
|
|
1586
|
+
}
|
|
1587
|
+
/**
|
|
1588
|
+
* Put evidence that reconciles other evidence ahead of the evidence it
|
|
1589
|
+
* reconciles. Readings of the same moment disagree, and the index resolves
|
|
1590
|
+
* those disagreements by checking each one against the whole timeline. Once
|
|
1591
|
+
* it has, the losing reading is still in the index and can still be
|
|
1592
|
+
* retrieved; leading with the resolution keeps an answer from being written
|
|
1593
|
+
* from a reading the index already set aside.
|
|
1594
|
+
*/
|
|
1595
|
+
function reconciledFirst(hits) {
|
|
1596
|
+
const reconciled = (hit) => hit.evidence.source?.provider === 'video-intelligence-index';
|
|
1597
|
+
return [...hits.filter(reconciled), ...hits.filter((hit) => !reconciled(hit))];
|
|
1598
|
+
}
|
|
1599
|
+
/**
|
|
1600
|
+
* A claim of this shape is settled by reading across records rather than by
|
|
1601
|
+
* finding one that states it: a conclusion follows a trail of states, a count
|
|
1602
|
+
* accumulates, a comparison needs both sides.
|
|
1603
|
+
*/
|
|
1604
|
+
function isCrossEvidenceClaim(plan) {
|
|
1605
|
+
return plan.kinds.some((kind) => ['outcome', 'state-change', 'comparison', 'counting', 'computation'].includes(kind));
|
|
1606
|
+
}
|
|
1607
|
+
/** Enough separate moments to read a trail rather than one lucky record. */
|
|
1608
|
+
const CORROBORATION_FLOOR = 2;
|
|
1609
|
+
/**
|
|
1610
|
+
* A "who" answer needs more than a positional descriptor such as "the right
|
|
1611
|
+
* side". The reconciled participant record is the index's identity anchor;
|
|
1612
|
+
* require that anchored name to also occur in the local answer trail. This is
|
|
1613
|
+
* language-agnostic because both strings come from the same source index.
|
|
1614
|
+
*/
|
|
1615
|
+
function trailHasRequiredIdentity(hits, trail, plan) {
|
|
1616
|
+
if (!plan.requiresIdentityContext)
|
|
1617
|
+
return true;
|
|
1618
|
+
const trailText = trail
|
|
1619
|
+
.map((hit) => evidenceText(hit.evidence.payload))
|
|
1620
|
+
.join('\n')
|
|
1621
|
+
.toLocaleLowerCase();
|
|
1622
|
+
return (identityAnchorNames(hits).some((name) => trailText.includes(name)) ||
|
|
1623
|
+
trailHasStableSourceLabels(trail));
|
|
1624
|
+
}
|
|
1625
|
+
/**
|
|
1626
|
+
* A repeated source label can identify a side or subject even when it is not a
|
|
1627
|
+
* person's name. This establishes the answer at that label's granularity; it
|
|
1628
|
+
* never licenses converting a position or label into an unshown personal name.
|
|
1629
|
+
*/
|
|
1630
|
+
function trailHasStableSourceLabels(trail) {
|
|
1631
|
+
const labelSets = chronological(trail)
|
|
1632
|
+
.filter((hit) => hit.evidence.modality === 'computed' &&
|
|
1633
|
+
hit.evidence.source?.provider === 'video-intelligence-index' &&
|
|
1634
|
+
/^(?:Reconciled|Indexed)\s+state:/i.test(evidenceText(hit.evidence.payload).trim()))
|
|
1635
|
+
.map((hit) => {
|
|
1636
|
+
const account = evidenceText(hit.evidence.payload).replace(/^(?:Reconciled|Indexed)\s+state:\s*/i, '');
|
|
1637
|
+
// A reconciled multi-subject state serializes its independently bound
|
|
1638
|
+
// entries as separate clauses. Free prose such as "the left label..."
|
|
1639
|
+
// is still a locator and must not become an identity binding.
|
|
1640
|
+
if (!/[,;،؛]/u.test(account))
|
|
1641
|
+
return new Set();
|
|
1642
|
+
return new Set((account.match(/[\p{Letter}\p{Mark}]+/gu) ?? [])
|
|
1643
|
+
.map((token) => token.normalize('NFKC').toLocaleLowerCase())
|
|
1644
|
+
.filter((token) => token.length >= 4));
|
|
1645
|
+
})
|
|
1646
|
+
.filter((labels) => labels.size > 0);
|
|
1647
|
+
if (labelSets.length < 2)
|
|
1648
|
+
return false;
|
|
1649
|
+
return labelSets.some((labels, index) => labelSets.slice(index + 1).some((other) => [...labels].some((label) => other.has(label))));
|
|
1650
|
+
}
|
|
1651
|
+
function identityAnchorNames(hits) {
|
|
1652
|
+
return hits.flatMap((hit) => {
|
|
1653
|
+
const text = evidenceText(hit.evidence.payload);
|
|
1654
|
+
return [
|
|
1655
|
+
...text.matchAll(/(?:Reconciled|Indexed) participant:\s*([^—\n]{1,120})/gi),
|
|
1656
|
+
...text.matchAll(/^Present:\s*([^—\n]{1,120})/gim),
|
|
1657
|
+
]
|
|
1658
|
+
.map((match) => match[1]?.trim().toLocaleLowerCase())
|
|
1659
|
+
.filter((name) => Boolean(name));
|
|
1660
|
+
});
|
|
1661
|
+
}
|
|
1662
|
+
function answeringRule(assessment, plan, analysisUnavailable) {
|
|
1663
|
+
const hasTrail = assessment.corroborating.length >= CORROBORATION_FLOOR;
|
|
1664
|
+
if (plan.requiresIdentityContext === true &&
|
|
1665
|
+
!plan.subjectName &&
|
|
1666
|
+
plan.kinds.includes('person-attribute') &&
|
|
1667
|
+
assessment.established.length > 0) {
|
|
1668
|
+
return 'The evidence distinguishes one or more visible subjects by source-supported attributes, but a personal name is not automatically established by appearance. Give every supported distinction (using an on-screen label, position, role, or other source-supported visual descriptor when needed), and separately say which personal names could not be grounded. Do not discard the established attributes merely because the names are unresolved. Describe only the people and attributes the returned evidence actually distinguishes; do not imply a complete roster when the source does not establish one.';
|
|
1669
|
+
}
|
|
1670
|
+
if (assessment.establishedByTrail) {
|
|
1671
|
+
return 'The answer is established by multiple timestamped observations read together. Synthesize their chronological trail directly, preserve the identifying details and supporting times, and do not describe it as unconfirmed merely because no single observation states the whole conclusion. Answer at the source label or role granularity it establishes; never replace a side or role label with a personal name unless the evidence explicitly binds them.';
|
|
1672
|
+
}
|
|
1673
|
+
if (!assessment.sufficient && plan.requiresIdentityContext) {
|
|
1674
|
+
return analysisUnavailable
|
|
1675
|
+
? 'The recorded states establish part of the result, but the closer source check could not complete and the evidence does not bind that result to the requested identity. Give the supported state or result with its timestamps, name the missing identity link, and do not infer a person from a side, label, or nearby name.'
|
|
1676
|
+
: 'The returned states establish part of the result but do not bind it to the requested identity. Give only the supported part and do not infer a person from a side, label, or nearby name.';
|
|
1677
|
+
}
|
|
1678
|
+
if (analysisUnavailable && !assessment.sufficient) {
|
|
1679
|
+
return hasTrail
|
|
1680
|
+
? 'A closer look at the source could not run just now, so these are its recorded observations. Read them in time order and give the answer they lead to, with the timestamps it rests on and how confident you are; say once that this is what the recording showed rather than something re-checked. Do not say the source does not show this -- if the trail settles only part of the question, give that part and name what is missing.'
|
|
1681
|
+
: 'These are recorded observations of the source, but a closer look at the moments they point to could not run just now. Answer from them, and say once that this is what the recording showed rather than something re-checked.';
|
|
1682
|
+
}
|
|
1683
|
+
if (!assessment.sufficient && isCrossEvidenceClaim(plan) && hasTrail) {
|
|
1684
|
+
// The claim was not settled by any single record, which is the normal case
|
|
1685
|
+
// for this shape of question -- not a sign the source is silent. Declining
|
|
1686
|
+
// here is what turned an indexed, answerable recording into "I could not
|
|
1687
|
+
// confirm that", so the instruction is to reason across the trail and be
|
|
1688
|
+
// explicit about how far it goes.
|
|
1689
|
+
return 'No single observation states this outright, which is expected for a claim of this shape. Read across the returned observations in time order and give the answer they lead to, with the timestamps it rests on and how confident you are. Do not say the source does not show this -- say what the observations establish and where they stop short.';
|
|
1690
|
+
}
|
|
1691
|
+
if (!assessment.sufficient) {
|
|
1692
|
+
return 'The returned evidence does not settle this. Say which part is established and which part the source did not show, rather than inferring the rest.';
|
|
1693
|
+
}
|
|
1694
|
+
if (assessment.needsBreadth) {
|
|
1695
|
+
return plan.kinds.includes('state-change')
|
|
1696
|
+
? 'These are timestamped source observations selected across the video. Give the account in their order, keep their timestamps for specific claims, and say where the source is silent rather than filling the gap.'
|
|
1697
|
+
: 'The evidence is distributed across the source. Synthesize the requested complete list or overview, deduplicate repeated items, preserve timestamps for specific claims, and state any explicit coverage limitation without exposing the evidence-gathering process.';
|
|
1698
|
+
}
|
|
1699
|
+
return 'Answer only from the returned source evidence.';
|
|
1700
|
+
}
|
|
1701
|
+
function chronological(hits) {
|
|
1702
|
+
return [...hits].sort((left, right) => left.evidence.timeRange.startSecs - right.evidence.timeRange.startSecs ||
|
|
1703
|
+
left.evidence.timeRange.endSecs - right.evidence.timeRange.endSecs);
|
|
1704
|
+
}
|
|
1705
|
+
function isInspectionFailure(value) {
|
|
1706
|
+
if (!value || typeof value !== 'object')
|
|
1707
|
+
return false;
|
|
1708
|
+
const result = value;
|
|
1709
|
+
return result.success === false && typeof result.error === 'string' && result.error.length > 0;
|
|
1710
|
+
}
|
|
1711
|
+
function isServiceBusy(value) {
|
|
1712
|
+
if (!value || typeof value !== 'object')
|
|
1713
|
+
return false;
|
|
1714
|
+
return value.serviceBusy === true;
|
|
1715
|
+
}
|
|
1716
|
+
function citationSurface(asset, context, evidence, primaryTimestampSecs) {
|
|
1717
|
+
const sourceUrl = `/api/media/${encodeURIComponent(asset.id)}${context.projectId ? `?projectId=${encodeURIComponent(context.projectId)}` : ''}`;
|
|
1718
|
+
return {
|
|
1719
|
+
kind: 'citations',
|
|
1720
|
+
title: asset.fileName ?? 'Video',
|
|
1721
|
+
fileName: asset.fileName ?? 'Video',
|
|
1722
|
+
mediaType: asset.type === 'audio' ? 'audio' : 'video',
|
|
1723
|
+
mediaUrl: sourceUrl,
|
|
1724
|
+
primaryTimestampSecs,
|
|
1725
|
+
// Keep citations inspectable without surfacing raw transcription or
|
|
1726
|
+
// frame-extraction text in the conversation.
|
|
1727
|
+
items: evidence.map((item) => ({
|
|
1728
|
+
label: evidenceLabel(item.modality),
|
|
1729
|
+
timestampSecs: item.timeRange.startSecs,
|
|
1730
|
+
seekUrl: `${sourceUrl}${sourceUrl.includes('?') ? '&' : '?'}t=${Math.floor(item.timeRange.startSecs)}`,
|
|
1731
|
+
})),
|
|
1732
|
+
};
|
|
1733
|
+
}
|
|
1734
|
+
function evidenceLabel(modality) {
|
|
1735
|
+
if (modality === 'transcript')
|
|
1736
|
+
return 'Speech';
|
|
1737
|
+
if (modality === 'ocr')
|
|
1738
|
+
return 'On-screen text';
|
|
1739
|
+
if (modality === 'visual')
|
|
1740
|
+
return 'Video';
|
|
1741
|
+
if (modality === 'computed')
|
|
1742
|
+
return 'Timeline';
|
|
1743
|
+
return 'Source';
|
|
1744
|
+
}
|
|
1745
|
+
function createInspector(fetcher) {
|
|
1746
|
+
return async function inspectVideoKnowledge(input, context) {
|
|
1747
|
+
if (!context.origin)
|
|
1748
|
+
return {
|
|
1749
|
+
success: false,
|
|
1750
|
+
error: 'Source inspection is unavailable without a request origin.',
|
|
1751
|
+
};
|
|
1752
|
+
if (!isValidInspectionInput(input))
|
|
1753
|
+
return { success: false, error: 'The requested inspection range is invalid.' };
|
|
1754
|
+
const url = new URL('/api/media/inspect', context.origin);
|
|
1755
|
+
if (context.projectId)
|
|
1756
|
+
url.searchParams.set('projectId', context.projectId);
|
|
1757
|
+
const chunks = splitRange(input.startSecs, input.endSecs, MAX_INSPECTION_CHUNK_SECS).reverse();
|
|
1758
|
+
let lastResult = { error: 'No inspectable range was provided.' };
|
|
1759
|
+
let lastOk = false;
|
|
1760
|
+
for (const chunk of chunks) {
|
|
1761
|
+
// The server has its own worker deadline, but a network socket or a
|
|
1762
|
+
// wedged control-plane request can otherwise outlive it indefinitely.
|
|
1763
|
+
// Keep a chat turn responsive even when that happens.
|
|
1764
|
+
const timeoutMs = Math.max(1, Math.floor(input.maxWaitMs ?? INTERACTIVE_INSPECTION_BUDGET_MS));
|
|
1765
|
+
try {
|
|
1766
|
+
const response = await fetcher(url, {
|
|
1767
|
+
method: 'POST',
|
|
1768
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1769
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
1770
|
+
body: JSON.stringify({
|
|
1771
|
+
mediaAssetId: input.mediaAssetId,
|
|
1772
|
+
startSecs: chunk.startSecs,
|
|
1773
|
+
endSecs: chunk.endSecs,
|
|
1774
|
+
purpose: input.purpose,
|
|
1775
|
+
queryId: `${input.queryId}:${chunk.startSecs}`.slice(0, 128),
|
|
1776
|
+
question: input.question ?? input.queryId,
|
|
1777
|
+
maxFrames: input.maxFrames,
|
|
1778
|
+
analysisMode: input.analysisMode,
|
|
1779
|
+
knownEntities: input.knownEntities,
|
|
1780
|
+
continuousSequence: input.continuousSequence,
|
|
1781
|
+
includeSpeech: input.includeSpeech,
|
|
1782
|
+
maxWaitMs: input.maxWaitMs,
|
|
1783
|
+
toolCallId: context.toolCallId,
|
|
1784
|
+
}),
|
|
1785
|
+
});
|
|
1786
|
+
lastResult = await response
|
|
1787
|
+
.json()
|
|
1788
|
+
.catch(() => ({ error: 'Inspection returned an invalid response.' }));
|
|
1789
|
+
lastOk = response.ok;
|
|
1790
|
+
if (lastOk && Array.isArray(lastResult.evidence) && lastResult.evidence.length > 0)
|
|
1791
|
+
break;
|
|
1792
|
+
}
|
|
1793
|
+
catch (error) {
|
|
1794
|
+
const timedOut = error instanceof DOMException && error.name === 'TimeoutError';
|
|
1795
|
+
return {
|
|
1796
|
+
success: false,
|
|
1797
|
+
error: timedOut
|
|
1798
|
+
? 'Video analysis did not finish within the interactive response budget.'
|
|
1799
|
+
: `Video analysis could not be reached: ${error instanceof Error ? error.message : 'unknown error'}`,
|
|
1800
|
+
};
|
|
1801
|
+
}
|
|
1802
|
+
}
|
|
1803
|
+
return lastOk ? { success: true, ...lastResult } : { success: false, ...lastResult };
|
|
1804
|
+
};
|
|
1805
|
+
}
|
|
1806
|
+
function evenlySpaced(items, limit) {
|
|
1807
|
+
if (limit <= 0)
|
|
1808
|
+
return [];
|
|
1809
|
+
if (items.length <= limit)
|
|
1810
|
+
return items;
|
|
1811
|
+
if (limit === 1)
|
|
1812
|
+
return [items[0]];
|
|
1813
|
+
return Array.from({ length: limit }, (_, index) => items[Math.round((index * (items.length - 1)) / (limit - 1))]);
|
|
1814
|
+
}
|
|
1815
|
+
function mergeHits(...groups) {
|
|
1816
|
+
const seen = new Set();
|
|
1817
|
+
return groups.flat().filter((hit) => {
|
|
1818
|
+
if (seen.has(hit.evidence.id))
|
|
1819
|
+
return false;
|
|
1820
|
+
seen.add(hit.evidence.id);
|
|
1821
|
+
return true;
|
|
1822
|
+
});
|
|
1823
|
+
}
|
|
1824
|
+
function uniqueRanges(ranges) {
|
|
1825
|
+
const unique = [];
|
|
1826
|
+
for (const range of ranges) {
|
|
1827
|
+
if (!range || range.endSecs <= range.startSecs)
|
|
1828
|
+
continue;
|
|
1829
|
+
if (unique.some((existing) => Math.abs(existing.startSecs - range.startSecs) < 10))
|
|
1830
|
+
continue;
|
|
1831
|
+
unique.push(range);
|
|
1832
|
+
}
|
|
1833
|
+
return unique;
|
|
1834
|
+
}
|
|
1835
|
+
function inspectionPurpose(kinds) {
|
|
1836
|
+
// OCR stays on for questions that turn on an exact visible value, so the
|
|
1837
|
+
// reader has an independent reading of the text rather than being its only
|
|
1838
|
+
// interpreter.
|
|
1839
|
+
if (kinds.includes('exact-ocr') || kinds.includes('state-change')) {
|
|
1840
|
+
return 'high-res-ocr';
|
|
1841
|
+
}
|
|
1842
|
+
if (kinds.includes('counting'))
|
|
1843
|
+
return 'count';
|
|
1844
|
+
if (kinds.includes('comparison'))
|
|
1845
|
+
return 'compare';
|
|
1846
|
+
return 'verify-visual';
|
|
1847
|
+
}
|
|
1848
|
+
function isDirectVerdict(hit, question, durationSecs) {
|
|
1849
|
+
const payload = evidenceText(hit.evidence.payload);
|
|
1850
|
+
const questionMatch = payload.match(/Claim question:\s*([^\n"]+)/i);
|
|
1851
|
+
const requestedRanges = temporalQuestionRanges(question, durationSecs);
|
|
1852
|
+
const asksIndividualNames = /\b(?:player|person|people|participant|speaker|presenter|guest)s?\b[^\n]*\bname|\bname(?:s)?\b[^\n]*\b(?:player|person|people|participant|speaker|presenter|guest)s?\b|(?:اسم|أسماء).{0,40}(?:لاعب|لاعبين|شخص|أشخاص|مشارك)/iu.test(question);
|
|
1853
|
+
const claimAnswer = payload.match(/Claim answer:\s*([^\n]+)/i)?.[1] ?? '';
|
|
1854
|
+
const substitutesCollectiveForPerson = asksIndividualNames &&
|
|
1855
|
+
/\b(?:team|group|organization|organisation|company|side)s?\s+names?\b|(?:أسماء\s+(?:الفرق|المجموعات|المنظمات))/iu.test(claimAnswer);
|
|
1856
|
+
const isInRequestedRange = requestedRanges.length === 0 ||
|
|
1857
|
+
requestedRanges.some((range) => hit.evidence.timeRange.startSecs < range.endSecs &&
|
|
1858
|
+
hit.evidence.timeRange.endSecs > range.startSecs);
|
|
1859
|
+
return (payload.includes('Claim verdict: direct') &&
|
|
1860
|
+
questionMatch !== null &&
|
|
1861
|
+
isInRequestedRange &&
|
|
1862
|
+
!substitutesCollectiveForPerson &&
|
|
1863
|
+
// A bounded reader receives the user's question plus a concise inspection
|
|
1864
|
+
// instruction. Its direct claim remains the same question, even though
|
|
1865
|
+
// that extra guidance lowers token overlap.
|
|
1866
|
+
// A direct claim is reusable only for the same question (or a very close
|
|
1867
|
+
// rewording). A looser threshold can reuse an opening-scene description
|
|
1868
|
+
// for a different attribute question simply because both mention the
|
|
1869
|
+
// same timestamp. That suppresses the bounded fallback precisely when
|
|
1870
|
+
// fresh visual analysis is required.
|
|
1871
|
+
questionSimilarity(questionMatch[1], question) >= 0.6);
|
|
1872
|
+
}
|
|
1873
|
+
function answerEstablishedHits(hits, question, kinds, watched = { ranges: [], since: '' }, durationSecs) {
|
|
1874
|
+
const verified = hits.filter((hit) => isDirectVerdict(hit, question, durationSecs));
|
|
1875
|
+
if (verified.length > 0)
|
|
1876
|
+
return verified;
|
|
1877
|
+
// Evidence recorded *by* watching a range for this question answers it by
|
|
1878
|
+
// provenance: that is the whole reason the range was watched. Matching on
|
|
1879
|
+
// the reader echoing the question back is not a sound substitute -- readers
|
|
1880
|
+
// reword it, and a rewording is not a failure to answer. This is narrower
|
|
1881
|
+
// than "evidence inside the range": evidence that was already in the index
|
|
1882
|
+
// before the look began was not gathered for this question, so it is held
|
|
1883
|
+
// to the ordinary bar. Anything that read the range and said it does not
|
|
1884
|
+
// settle the claim is excluded.
|
|
1885
|
+
const fromWatching = watched.since
|
|
1886
|
+
? hits.filter((hit) => hit.evidence.createdAt >= watched.since &&
|
|
1887
|
+
watched.ranges.some((range) => hit.evidence.timeRange.startSecs < range.endSecs &&
|
|
1888
|
+
hit.evidence.timeRange.endSecs > range.startSecs) &&
|
|
1889
|
+
['visual', 'transcript', 'ocr'].includes(hit.evidence.modality) &&
|
|
1890
|
+
!hasNegativeVerdictForQuestion(hit, question))
|
|
1891
|
+
: [];
|
|
1892
|
+
// Watching a moment produces two kinds of record: an account of what the
|
|
1893
|
+
// moment showed, and raw signal from it -- a single word read off the
|
|
1894
|
+
// screen, a note that some text was present. The account answers the
|
|
1895
|
+
// question; the raw signal only corroborates it, and there is far more of
|
|
1896
|
+
// it. Ordered the other way, an answer gets written from stray words.
|
|
1897
|
+
const accounts = fromWatching.filter(isAccountOfMoment);
|
|
1898
|
+
if (accounts.length > 0)
|
|
1899
|
+
return [...accounts, ...fromWatching.filter((hit) => !isAccountOfMoment(hit))];
|
|
1900
|
+
// Direct source observations already materialized during indexing should
|
|
1901
|
+
// answer straightforward speech, visible-text, and visual-fact questions
|
|
1902
|
+
// immediately. Aggregates, comparisons, identity attribution, and
|
|
1903
|
+
// conclusions stay stricter because they require reasoning across
|
|
1904
|
+
// observations rather than reading one source-backed fact.
|
|
1905
|
+
const requiresCrossEvidenceReasoning = kinds.some((kind) => [
|
|
1906
|
+
'outcome',
|
|
1907
|
+
'state-change',
|
|
1908
|
+
'comparison',
|
|
1909
|
+
'counting',
|
|
1910
|
+
'computation',
|
|
1911
|
+
'person-attribute',
|
|
1912
|
+
'visual-fact',
|
|
1913
|
+
].includes(kind));
|
|
1914
|
+
if (requiresCrossEvidenceReasoning)
|
|
1915
|
+
return [];
|
|
1916
|
+
return hits.filter((hit) => isIndexedDirectObservation(hit, question));
|
|
1917
|
+
}
|
|
1918
|
+
/**
|
|
1919
|
+
* Whether a record recounts what a moment showed, rather than carrying raw
|
|
1920
|
+
* signal from it. A reader's description of a clip and a spoken sentence both
|
|
1921
|
+
* recount; a single word lifted off the screen, or a note that some text was
|
|
1922
|
+
* present, do not -- they are corroboration, and there are many more of them.
|
|
1923
|
+
*/
|
|
1924
|
+
function isAccountOfMoment(hit) {
|
|
1925
|
+
// 'computed' covers the index's own reconciled account, which recounts a
|
|
1926
|
+
// moment after cross-checking every reading of it against the timeline.
|
|
1927
|
+
if (!['visual', 'transcript', 'computed'].includes(hit.evidence.modality))
|
|
1928
|
+
return false;
|
|
1929
|
+
const payload = hit.evidence.payload;
|
|
1930
|
+
const text = payload && typeof payload === 'object' && !Array.isArray(payload)
|
|
1931
|
+
? payload.text
|
|
1932
|
+
: payload;
|
|
1933
|
+
// A typed subject/property/value locates a display; it does not describe it.
|
|
1934
|
+
return typeof text === 'string' && text.trim().length > 0;
|
|
1935
|
+
}
|
|
1936
|
+
function isIndexedDirectObservation(hit, question) {
|
|
1937
|
+
if (!['transcript', 'ocr', 'visual'].includes(hit.evidence.modality))
|
|
1938
|
+
return false;
|
|
1939
|
+
const confidence = typeof hit.evidence.confidence === 'object' && hit.evidence.confidence !== null
|
|
1940
|
+
? Number(hit.evidence.confidence.score)
|
|
1941
|
+
: 0;
|
|
1942
|
+
if (!Number.isFinite(confidence) || confidence < 0.45)
|
|
1943
|
+
return false;
|
|
1944
|
+
if (hasNegativeVerdictForQuestion(hit, question))
|
|
1945
|
+
return false;
|
|
1946
|
+
// Retrieval can return a high-quality neighbouring scene even when it does
|
|
1947
|
+
// not contain the fact the user asked for. Treat an indexed observation as
|
|
1948
|
+
// answer-level evidence only when it shares a meaningful query term or the
|
|
1949
|
+
// hybrid retriever explicitly marked it as a semantic match. Otherwise the
|
|
1950
|
+
// bounded live reader gets the opportunity to inspect the candidate range.
|
|
1951
|
+
const semanticMatch = Number(hit.components?.semantic ?? 0) > 0;
|
|
1952
|
+
if (semanticMatch)
|
|
1953
|
+
return true;
|
|
1954
|
+
const questionTerms = new Set(question
|
|
1955
|
+
.normalize('NFKC')
|
|
1956
|
+
.toLocaleLowerCase()
|
|
1957
|
+
.match(/[\p{Letter}\p{Number}][\p{Letter}\p{Number}\p{Mark}_-]*/gu)
|
|
1958
|
+
?.filter((term) => term.length > 3) ?? []);
|
|
1959
|
+
if (questionTerms.size === 0)
|
|
1960
|
+
return false;
|
|
1961
|
+
const sourceTerms = new Set(evidenceText(hit.evidence.payload)
|
|
1962
|
+
.normalize('NFKC')
|
|
1963
|
+
.toLocaleLowerCase()
|
|
1964
|
+
.match(/[\p{Letter}\p{Number}][\p{Letter}\p{Number}\p{Mark}_-]*/gu) ?? []);
|
|
1965
|
+
return [...questionTerms].some((term) => sourceTerms.has(term));
|
|
1966
|
+
}
|
|
1967
|
+
function hasNegativeVerdictForQuestion(hit, question) {
|
|
1968
|
+
const payload = evidenceText(hit.evidence.payload);
|
|
1969
|
+
const questionMatch = payload.match(/Claim question:\s*([^\n"]+)/i);
|
|
1970
|
+
return Boolean(/Claim verdict:\s*(?:partial|not-established)/i.test(payload) &&
|
|
1971
|
+
questionMatch !== null &&
|
|
1972
|
+
questionSimilarity(questionMatch[1], question) >= 0.35);
|
|
1973
|
+
}
|
|
1974
|
+
function hasNegativeVerdict(hit) {
|
|
1975
|
+
return /Claim verdict:\s*not-established/i.test(evidenceText(hit.evidence.payload));
|
|
1976
|
+
}
|
|
1977
|
+
function evidenceText(value) {
|
|
1978
|
+
if (typeof value === 'string')
|
|
1979
|
+
return value;
|
|
1980
|
+
if (Array.isArray(value))
|
|
1981
|
+
return value.map(evidenceText).join('\n');
|
|
1982
|
+
if (value && typeof value === 'object')
|
|
1983
|
+
return Object.values(value).map(evidenceText).join('\n');
|
|
1984
|
+
return '';
|
|
1985
|
+
}
|
|
1986
|
+
/**
|
|
1987
|
+
* Token overlap that lets a provider's faithful paraphrase reuse a verified
|
|
1988
|
+
* source pass, without letting two unrelated questions look alike.
|
|
1989
|
+
*
|
|
1990
|
+
* Terms are weighted by length. Counting them equally makes any two short
|
|
1991
|
+
* questions in the same language look similar, because the words they share
|
|
1992
|
+
* are the ones every question has -- "what", "was", "the" -- and those are
|
|
1993
|
+
* a large fraction of a short question's tokens. That is enough to make a
|
|
1994
|
+
* verified answer to one question get reused for a different one. Longer
|
|
1995
|
+
* terms carry the subject matter in every script, so weighting by length
|
|
1996
|
+
* keeps two questions apart when all they share is grammar, while still
|
|
1997
|
+
* matching a genuine rewording of the same one.
|
|
1998
|
+
*/
|
|
1999
|
+
function questionSimilarity(left, right) {
|
|
2000
|
+
const terms = (value) => new Set(value
|
|
2001
|
+
.normalize('NFKC')
|
|
2002
|
+
.toLocaleLowerCase()
|
|
2003
|
+
.match(/[\p{Letter}\p{Number}]+/gu) ?? []);
|
|
2004
|
+
const weigh = (values) => [...values].reduce((total, term) => total + term.length, 0);
|
|
2005
|
+
const leftTerms = terms(left);
|
|
2006
|
+
const rightTerms = terms(right);
|
|
2007
|
+
if (leftTerms.size === 0 || rightTerms.size === 0)
|
|
2008
|
+
return 0;
|
|
2009
|
+
const shared = new Set([...leftTerms].filter((term) => rightTerms.has(term)));
|
|
2010
|
+
return weigh(shared) / Math.max(weigh(leftTerms), weigh(rightTerms));
|
|
2011
|
+
}
|
|
2012
|
+
function toEvidence(hit, _plan) {
|
|
2013
|
+
return {
|
|
2014
|
+
id: hit.evidence.id,
|
|
2015
|
+
modality: hit.evidence.modality,
|
|
2016
|
+
timeRange: hit.evidence.timeRange,
|
|
2017
|
+
payload: hit.evidence.payload,
|
|
2018
|
+
confidence: hit.evidence.confidence,
|
|
2019
|
+
};
|
|
2020
|
+
}
|
|
2021
|
+
function stableQueryId(query) {
|
|
2022
|
+
let hash = 2166136261;
|
|
2023
|
+
for (const char of query.normalize('NFKC'))
|
|
2024
|
+
hash = Math.imul(hash ^ char.codePointAt(0), 16777619);
|
|
2025
|
+
return `evidence-${(hash >>> 0).toString(36)}`;
|
|
2026
|
+
}
|
|
2027
|
+
/**
|
|
2028
|
+
* Source resolution can append a selected file's title to the user's request.
|
|
2029
|
+
* Remove only a long trailing run made entirely from that title, leaving names
|
|
2030
|
+
* and other source-specific terms elsewhere in the actual question intact.
|
|
2031
|
+
*/
|
|
2032
|
+
function focusQuestion(question, fileName) {
|
|
2033
|
+
const normalized = question.normalize('NFKC').trim();
|
|
2034
|
+
if (!fileName)
|
|
2035
|
+
return normalized;
|
|
2036
|
+
const comparableTerm = (term) => term
|
|
2037
|
+
.toLocaleLowerCase()
|
|
2038
|
+
.replace(/ى/g, 'ي')
|
|
2039
|
+
.replace(/\p{Mark}/gu, '');
|
|
2040
|
+
const titleTerms = new Set(fileName
|
|
2041
|
+
.normalize('NFKC')
|
|
2042
|
+
.toLocaleLowerCase()
|
|
2043
|
+
.match(/[\p{Letter}\p{Number}][\p{Letter}\p{Number}\p{Mark}_-]*/gu)
|
|
2044
|
+
?.map(comparableTerm) ?? []);
|
|
2045
|
+
const tokens = [
|
|
2046
|
+
...normalized.matchAll(/[\p{Letter}\p{Number}][\p{Letter}\p{Number}\p{Mark}_-]*/gu),
|
|
2047
|
+
];
|
|
2048
|
+
let suffixStart = normalized.length;
|
|
2049
|
+
let suffixTerms = 0;
|
|
2050
|
+
for (let index = tokens.length - 1; index >= 0; index -= 1) {
|
|
2051
|
+
const token = tokens[index];
|
|
2052
|
+
if (!titleTerms.has(comparableTerm(token[0])))
|
|
2053
|
+
break;
|
|
2054
|
+
suffixStart = token.index ?? suffixStart;
|
|
2055
|
+
suffixTerms += 1;
|
|
2056
|
+
}
|
|
2057
|
+
if (suffixTerms < 3)
|
|
2058
|
+
return normalized;
|
|
2059
|
+
const focused = normalized
|
|
2060
|
+
.slice(0, suffixStart)
|
|
2061
|
+
.replace(/[\s|:;,.،؛\-–—]+$/u, '')
|
|
2062
|
+
.trim();
|
|
2063
|
+
return focused || normalized;
|
|
2064
|
+
}
|
|
2065
|
+
function isValidQueryInput(input) {
|
|
2066
|
+
return (typeof input.mediaAssetId === 'string' &&
|
|
2067
|
+
input.mediaAssetId.length > 0 &&
|
|
2068
|
+
typeof input.query === 'string' &&
|
|
2069
|
+
input.query.trim().length > 0);
|
|
2070
|
+
}
|
|
2071
|
+
function isValidInspectionInput(input) {
|
|
2072
|
+
return (typeof input.mediaAssetId === 'string' &&
|
|
2073
|
+
input.mediaAssetId.length > 0 &&
|
|
2074
|
+
Number.isFinite(input.startSecs) &&
|
|
2075
|
+
Number.isFinite(input.endSecs) &&
|
|
2076
|
+
input.startSecs >= 0 &&
|
|
2077
|
+
input.endSecs > input.startSecs &&
|
|
2078
|
+
['verify-visual', 'high-res-ocr', 'compare', 'count', 'track', 'code'].includes(input.purpose) &&
|
|
2079
|
+
typeof input.queryId === 'string' &&
|
|
2080
|
+
input.queryId.length > 0);
|
|
2081
|
+
}
|
|
2082
|
+
function splitRange(startSecs, endSecs, maxChunkSecs) {
|
|
2083
|
+
const chunks = [];
|
|
2084
|
+
for (let cursor = startSecs; cursor < endSecs; cursor += maxChunkSecs)
|
|
2085
|
+
chunks.push({ startSecs: cursor, endSecs: Math.min(endSecs, cursor + maxChunkSecs) });
|
|
2086
|
+
return chunks;
|
|
2087
|
+
}
|