@larkup/workspace-server 0.2.6 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-types/app/api/chat/route.d.ts.map +1 -1
- package/dist-types/lib/chat/conversation-memory.d.ts +9 -1
- package/dist-types/lib/chat/conversation-memory.d.ts.map +1 -1
- package/dist-types/lib/chat/tool-context.d.ts +4 -1
- package/dist-types/lib/chat/tool-context.d.ts.map +1 -1
- package/dist-types/lib/chat/video-query-planning.d.ts +15 -0
- package/dist-types/lib/chat/video-query-planning.d.ts.map +1 -0
- package/package.json +6 -5
- package/src/app/api/chat/route.ts +83 -27
- package/src/app/api/chat/tools.ts +2 -2
- package/src/lib/chat/conversation-memory.ts +49 -6
- package/src/lib/chat/tool-context.ts +47 -3
- package/src/lib/chat/video-query-planning.ts +44 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"route.d.ts","sourceRoot":"","sources":["../../../../src/app/api/chat/route.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"route.d.ts","sourceRoot":"","sources":["../../../../src/app/api/chat/route.ts"],"names":[],"mappings":"AAqHA,eAAO,MAAM,WAAW,MAAM,CAAC;AAiuB/B,wBAAsB,IAAI,CAAC,GAAG,EAAE,OAAO,qBA69CtC"}
|
|
@@ -9,7 +9,8 @@ export type ExactGroundedAnswer = {
|
|
|
9
9
|
sourceScopeFingerprint: string;
|
|
10
10
|
};
|
|
11
11
|
export type ReusableImageEvidence = {
|
|
12
|
-
imageUrl
|
|
12
|
+
imageUrl?: string;
|
|
13
|
+
documentId?: string;
|
|
13
14
|
pageNumber?: number;
|
|
14
15
|
index?: number;
|
|
15
16
|
title?: string;
|
|
@@ -57,6 +58,13 @@ export declare function extractConversationEvidence(messages: readonly ChatMessa
|
|
|
57
58
|
*/
|
|
58
59
|
export declare function isTabularFollowUp(text: string, evidence: ConversationEvidence): boolean;
|
|
59
60
|
export declare function isImagePreviewFollowUp(text: string, evidence: ConversationEvidence): boolean;
|
|
61
|
+
/** Reuse the exact prior source, preferring a fresh PDF-page render over a stale derivative. */
|
|
62
|
+
export declare function imagePreviewInput(evidence: ConversationEvidence): {
|
|
63
|
+
documentId: string;
|
|
64
|
+
pageNumber: number;
|
|
65
|
+
} | {
|
|
66
|
+
imageUrl: string;
|
|
67
|
+
} | undefined;
|
|
60
68
|
/**
|
|
61
69
|
* A source is selected by structured evidence from the immediately preceding
|
|
62
70
|
* turn. Keep it until a later source-selection event replaces it; never infer
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"conversation-memory.d.ts","sourceRoot":"","sources":["../../../src/lib/chat/conversation-memory.ts"],"names":[],"mappings":"AAeA,KAAK,WAAW,GAAG;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,sBAAsB,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"conversation-memory.d.ts","sourceRoot":"","sources":["../../../src/lib/chat/conversation-memory.ts"],"names":[],"mappings":"AAeA,KAAK,WAAW,GAAG;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,sBAAsB,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACrC,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,0FAA0F;AAC1F,wBAAgB,iCAAiC,CAC/C,QAAQ,EAAE,uBAAuB,GAAG,SAAS,EAC7C,mBAAmB,EAAE,WAAW,CAAC,MAAM,CAAC,GACvC,OAAO,CAET;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClD,MAAM,EAAE,qBAAqB,EAAE,CAAC;IAChC,yFAAyF;IACzF,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,6FAA6F;IAC7F,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,+EAA+E;IAC/E,OAAO,CAAC,EAAE,uBAAuB,CAAC;CACnC,CAAC;AAoFF;;;;GAIG;AACH,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,SAAS,WAAW,EAAE,EAChC,eAAe,EAAE,MAAM,GACtB,mBAAmB,GAAG,SAAS,CA0DjC;AA4ED,gFAAgF;AAChF,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,SAAS,WAAW,EAAE,GAC/B,uBAAuB,GAAG,SAAS,CAUrC;AAED,sFAAsF;AACtF,wBAAgB,iCAAiC,CAC/C,MAAM,EAAE,OAAO,EAAE,EACjB,OAAO,EAAE,MAAM,EAAE,GAChB,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAuBhC;AAYD;;;GAGG;AACH,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,SAAS,WAAW,EAAE,GAC/B,oBAAoB,CAmItB;AAmCD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,oBAAoB,GAAG,OAAO,CAavF;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,oBAAoB,GAAG,OAAO,CAK5F;AAED,gGAAgG;AAChG,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,oBAAoB,GAC7B;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAO/E;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,oBAAoB,GAAG,OAAO,CAE/F;AAUD;;;;GAIG;AACH,wBAAgB,iCAAiC,CAC/C,eAAe,EAAE,MAAM,EACvB,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAepB;AAED;;;;GAIG;AACH,wBAAgB,mCAAmC,CACjD,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,oBAAoB,GAC7B,MAAM,GAAG,SAAS,CAyBpB;AAED,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,oBAAoB,GAAG,MAAM,CAoCjF"}
|
|
@@ -38,7 +38,10 @@ export declare function formatLocatedObservedSubjectAnswer(value: unknown): stri
|
|
|
38
38
|
* asking a prose model to reinterpret explicit discrete moments as one
|
|
39
39
|
* continuous appearance or as an absence claim.
|
|
40
40
|
*/
|
|
41
|
-
export declare function formatObservedAppearanceAnswer(value: unknown
|
|
41
|
+
export declare function formatObservedAppearanceAnswer(value: unknown, investigation?: {
|
|
42
|
+
goal?: unknown;
|
|
43
|
+
recordSet?: unknown;
|
|
44
|
+
}): string | undefined;
|
|
42
45
|
/** Render a final two-sided numeric outcome without waiting on a slow prose model. */
|
|
43
46
|
export declare function formatOutcomeMediaAnswer(value: unknown, investigation: VideoInvestigationDirective): string | undefined;
|
|
44
47
|
/** Use the strongest bounded source reading directly when it already settled the question. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-context.d.ts","sourceRoot":"","sources":["../../../src/lib/chat/tool-context.ts"],"names":[],"mappings":"AAUA;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAmBpE;AAED,0EAA0E;AAC1E,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAgB/D;AAED,kFAAkF;AAClF,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAmBxE;AAmBD,oEAAoE;AACpE,wBAAgB,kCAAkC,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAuB7F;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CA+D1E;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CAyDrE;AAED;;;;;GAKG;AACH,wBAAgB,kCAAkC,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CA4KrF;AAED;;;;;;GAMG;AACH,wBAAgB,8BAA8B,
|
|
1
|
+
{"version":3,"file":"tool-context.d.ts","sourceRoot":"","sources":["../../../src/lib/chat/tool-context.ts"],"names":[],"mappings":"AAUA;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAmBpE;AAED,0EAA0E;AAC1E,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAgB/D;AAED,kFAAkF;AAClF,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAmBxE;AAmBD,oEAAoE;AACpE,wBAAgB,kCAAkC,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAuB7F;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CA+D1E;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CAyDrE;AAED;;;;;GAKG;AACH,wBAAgB,kCAAkC,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CA4KrF;AAED;;;;;;GAMG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,OAAO,EACd,aAAa,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GACtD,MAAM,GAAG,SAAS,CAwGpB;AAED,sFAAsF;AACtF,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,2BAA2B,GACzC,MAAM,GAAG,SAAS,CAyHpB;AAED,8FAA8F;AAC9F,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,SAAK,GAAG,MAAM,GAAG,SAAS,CAqE/F;AAED,8EAA8E;AAC9E,wBAAgB,+BAA+B,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CA2BvE;AAED,sGAAsG;AACtG,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CA0D/F;AAYD;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CA+IjG;AAED,gGAAgG;AAChG,wBAAgB,2BAA2B,CAAC,YAAY,EAAE,MAAM,mCAwC/D;AAkLD;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,SAAS,GAAG,EAAE,GAAG,GAAG,EAAE,CAapE;AAED,uFAAuF;AACvF,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,SAAS,GAAG,EAAE,GAAG,GAAG,EAAE,CAiB1E;AACD,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,4CAA4C,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { VideoInvestigationDirective } from '@larkup/core/video-knowledge/query-planner';
|
|
2
|
+
export interface VideoQuestionSignals {
|
|
3
|
+
hasPosition?: boolean;
|
|
4
|
+
connectsBeyondPosition?: boolean;
|
|
5
|
+
asksForSettledOutcome?: boolean;
|
|
6
|
+
asksForExhaustiveCoverage?: boolean;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Reconcile the planner directive with the independent question classifier.
|
|
10
|
+
* The classifier is deliberately language-neutral and has final authority over
|
|
11
|
+
* outcome requests, because a broad planner can otherwise mistake “who won?”
|
|
12
|
+
* for source synthesis and send the answer through appearance rendering.
|
|
13
|
+
*/
|
|
14
|
+
export declare function reconcileVideoInvestigation(planned: VideoInvestigationDirective | undefined, signals: VideoQuestionSignals | undefined): VideoInvestigationDirective | undefined;
|
|
15
|
+
//# sourceMappingURL=video-query-planning.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"video-query-planning.d.ts","sourceRoot":"","sources":["../../../src/lib/chat/video-query-planning.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,4CAA4C,CAAC;AAE9F,MAAM,WAAW,oBAAoB;IACnC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB,CAAC,EAAE,OAAO,CAAC;CACrC;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,OAAO,EAAE,2BAA2B,GAAG,SAAS,EAChD,OAAO,EAAE,oBAAoB,GAAG,SAAS,GACxC,2BAA2B,GAAG,SAAS,CAyBzC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@larkup/workspace-server",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.14",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"files": [
|
|
6
6
|
"src",
|
|
@@ -317,15 +317,16 @@
|
|
|
317
317
|
"@ai-sdk/mistral": "4.0.61",
|
|
318
318
|
"@ai-sdk/openai": "4.0.89",
|
|
319
319
|
"@ai-sdk/openai-compatible": "3.0.66",
|
|
320
|
-
"@larkup/core": "^0.8.
|
|
321
|
-
"@larkup/tool-doc-editor": "^0.3.
|
|
320
|
+
"@larkup/core": "^0.8.2",
|
|
321
|
+
"@larkup/tool-doc-editor": "^0.3.2",
|
|
322
|
+
"@napi-rs/canvas": "1.0.11",
|
|
322
323
|
"ai": "7.0.133",
|
|
323
324
|
"pdf-parse": "2.4.5",
|
|
324
325
|
"zod": "^4.4.3",
|
|
325
326
|
"mammoth": "^1.12.0",
|
|
326
|
-
"@larkup/marketplace": "0.3.
|
|
327
|
-
"@larkup/scraper": "0.2.1",
|
|
327
|
+
"@larkup/marketplace": "0.3.11",
|
|
328
328
|
"@larkup/sandbox": "0.1.8",
|
|
329
|
+
"@larkup/scraper": "0.2.2",
|
|
329
330
|
"@larkup/vector-stores": "0.1.27"
|
|
330
331
|
},
|
|
331
332
|
"devDependencies": {
|
|
@@ -41,12 +41,14 @@ import { getChatTools } from './tools';
|
|
|
41
41
|
import { gatewayProviderOptions } from '../../../lib/chat/gateway-fallbacks';
|
|
42
42
|
import { retrievalToolsForStep } from '../../../lib/chat/retrieval-routing';
|
|
43
43
|
import { collectExhaustiveVideoEvidencePages } from '../../../lib/chat/video-rag-routing';
|
|
44
|
+
import { reconcileVideoInvestigation } from '../../../lib/chat/video-query-planning';
|
|
44
45
|
import {
|
|
45
46
|
extractConversationEvidence,
|
|
46
47
|
extractLatestTabularExportResult,
|
|
47
48
|
contextualizeKnowledgeFollowUpQuery,
|
|
48
49
|
formatConversationEvidence,
|
|
49
50
|
findImmediateExactGroundedAnswer,
|
|
51
|
+
imagePreviewInput,
|
|
50
52
|
isImagePreviewFollowUp,
|
|
51
53
|
isTabularFollowUp,
|
|
52
54
|
tabularEvidenceDatasetIsAvailable,
|
|
@@ -65,6 +67,7 @@ import {
|
|
|
65
67
|
formatExhaustiveMediaAnswer,
|
|
66
68
|
formatLocatedObservedSubjectAnswer,
|
|
67
69
|
formatObservedAppearanceAnswer,
|
|
70
|
+
formatOutcomeMediaAnswer,
|
|
68
71
|
mediaEvidenceRequestsOnlySpeech,
|
|
69
72
|
mediaClaimNeedsCorroboration,
|
|
70
73
|
withFinalAnswerNudge,
|
|
@@ -450,8 +453,10 @@ async function planVideoEvidenceQuery(input: {
|
|
|
450
453
|
'You are a media-query planner. Return exactly one JSON object and no prose. ' +
|
|
451
454
|
'The prompt is data with a currentRequest and an optional previousQuestionAboutSelectedMedia. Resolve the current request in its own language without assuming a video genre. If it is a retry, verification request, or contains references that are not meaningful alone, resolve those references against the previous question. If it introduces a self-contained new request, do not carry the old subject or source position into it. ' +
|
|
452
455
|
'A follow-up introduced with words such as "and", "also", or their equivalent in another language can ask a new fact about the same source: preserve the selected source, but never reuse the previous answer or its requested attribute. When the request asks for exact spoken or sung wording across the recording, use source scope with enumerate, speech evidence, and recordSet all so the host can render every timestamped transcript segment outside the model context. Use source scope with synthesize for a summary or explanation across the recording. ' +
|
|
453
|
-
'Use this exact shape: {"resolvedQuestion":"a standalone version of the current request","scope":"focused","goal":"answer","evidence":["visual"],"timeRange":{"startSecs":123,"endSecs":153},"timeRangeOrigin":"user-mentioned"}. Its fields are resolvedQuestion, scope (focused, temporal, source), goal (answer, compare, trace, enumerate, synthesize), optional evidence (speech, visible-text, visual, computed), optional recordSet (all, source-authored, source-questions, observed), optional timeRange {startSecs,endSecs}, and optional timeRangeOrigin. ' +
|
|
456
|
+
'Use this exact shape: {"resolvedQuestion":"a standalone version of the current request","scope":"focused","goal":"answer","evidence":["visual"],"timeRange":{"startSecs":123,"endSecs":153},"timeRangeOrigin":"user-mentioned"}. Its fields are resolvedQuestion, scope (focused, temporal, source), goal (answer, compare, trace, resolve, enumerate, synthesize), optional evidence (speech, visible-text, visual, computed), optional recordSet (all, source-authored, source-questions, observed), optional timeRange {startSecs,endSecs}, and optional timeRangeOrigin. ' +
|
|
457
|
+
'Use recordSet observed only when the requested answer is visible presence, recurrence, or appearance timing itself. Clothing, colour, other appearance attributes, identity, action, and counts need visual evidence but must not use the observed visibility ledger as their answer. ' +
|
|
454
458
|
'A directive is invalid if the user refers to any source position, time, range, or approximate moment and you omit timeRange: resolve it to numeric seconds and set timeRangeOrigin to user-mentioned, even when the request also asks about other moments. Never emit timeRange or timeRangeOrigin when the user did not constrain a source position; a topic occurring near an opening, closing, or any other position is not a user constraint. When the request names a coarse source unit without a smaller boundary, cover that whole named unit rather than an arbitrary part of it. When the user asks how a visible subject or event at an explicit source position relates to appearances or recurrence elsewhere, use temporal plus trace, visual evidence, and recordSet observed while retaining that position. Use source plus synthesize with no timeRange whenever the answer must combine two or more source facts from unrestricted positions. Use source plus enumerate and recordSet source-questions for a complete inventory of source-authored questions and their answers. Use temporal for a relationship or development across moments; use focused only for one local moment or one self-contained fact. ' +
|
|
459
|
+
'Use temporal plus resolve when the requested fact is the settled or final outcome of a process, contest, decision, vote, test, or other sequence; this includes asking who won or what the final result was in any language. Do not confuse an entity mentioned in that request with a request to inventory its visual appearances. ' +
|
|
455
460
|
'Do not answer the question or add fields.',
|
|
456
461
|
320,
|
|
457
462
|
);
|
|
@@ -461,7 +466,7 @@ async function planVideoEvidenceQuery(input: {
|
|
|
461
466
|
// first planning attempt without introducing source- or language-specific
|
|
462
467
|
// routing rules in the host.
|
|
463
468
|
return await requestPlan(
|
|
464
|
-
'Return only JSON with resolvedQuestion, scope, goal, and optional recordSet/timeRange/timeRangeOrigin. Resolve an elliptical retry or reference against previousQuestionAboutSelectedMedia; otherwise keep currentRequest standalone. Select source/enumerate/source-questions for a complete question-and-answer inventory from a recording;
|
|
469
|
+
'Return only JSON with resolvedQuestion, scope, goal, and optional recordSet/timeRange/timeRangeOrigin. Resolve an elliptical retry or reference against previousQuestionAboutSelectedMedia; otherwise keep currentRequest standalone. Select source/enumerate/source-questions for a complete question-and-answer inventory from a recording; source/synthesize for an unrestricted explanation; temporal/resolve for a settled or final outcome such as who won; focused/answer for one local fact. Do not include a timeRange unless the resolved request contains a source position, and then set timeRangeOrigin to user-mentioned.',
|
|
465
470
|
180,
|
|
466
471
|
);
|
|
467
472
|
} catch {
|
|
@@ -486,6 +491,8 @@ async function userSpecifiedSourcePosition(input: {
|
|
|
486
491
|
| {
|
|
487
492
|
hasPosition: boolean;
|
|
488
493
|
connectsBeyondPosition: boolean;
|
|
494
|
+
asksForSettledOutcome: boolean;
|
|
495
|
+
asksForExhaustiveCoverage: boolean;
|
|
489
496
|
}
|
|
490
497
|
| undefined
|
|
491
498
|
> {
|
|
@@ -498,7 +505,7 @@ async function userSpecifiedSourcePosition(input: {
|
|
|
498
505
|
abortSignal: AbortSignal.timeout(8_000),
|
|
499
506
|
providerOptions: input.providerOptions,
|
|
500
507
|
system:
|
|
501
|
-
'Return exactly one JSON object and no prose: {"kind":"input-location"|"answer-timestamp"|"none","locator":string|null,"connectsBeyondPosition":boolean}. Use input-location only when the user explicitly constrains where in the provided source to look; locator must be the exact corresponding substring from the user request. Use answer-timestamp when the user asks for timestamps in the answer but does not specify where to look; locator is null. Use none when neither is present. Set connectsBeyondPosition true only when the request explicitly relates the thing at its named source position to another point or portion of the source, such as recurrence, reappearance, a later/earlier occurrence, change, or comparison; otherwise false. Example: "Explain a subject and include timestamps" => {"kind":"answer-timestamp","locator":null,"connectsBeyondPosition":false}. Example: "What happens at 14:00?" => {"kind":"input-location","locator":"14:00","connectsBeyondPosition":false}. Example: "Who is at 14:00 and where else do they appear?" => {"kind":"input-location","locator":"14:00","connectsBeyondPosition":true}. Example: "Give the full story from beginning to conclusion" => {"kind":"none","locator":null,"connectsBeyondPosition":false} unless the user asks specifically about the beginning or conclusion. Interpret every language without assuming a source genre.',
|
|
508
|
+
'Return exactly one JSON object and no prose: {"kind":"input-location"|"answer-timestamp"|"none","locator":string|null,"connectsBeyondPosition":boolean,"asksForSettledOutcome":boolean,"asksForExhaustiveCoverage":boolean}. Use input-location only when the user explicitly constrains where in the provided source to look; locator must be the exact corresponding substring from the user request. Use answer-timestamp when the user asks for timestamps in the answer but does not specify where to look; locator is null. Use none when neither is present. Set connectsBeyondPosition true only when the request explicitly relates the thing at its named source position to another point or portion of the source, such as recurrence, reappearance, a later/earlier occurrence, change, or comparison; otherwise false. Set asksForSettledOutcome true only when the request asks for the final or settled result of a process, contest, decision, vote, test, or other sequence, including who won in any language; it overrides a broad summary classification. Set asksForExhaustiveCoverage true only when the user explicitly asks to enumerate all or every relevant record across the complete recording, such as a complete start-to-finish timeline, every spoken statement, or every visual and spoken event. A full story, overview, or summary is source-wide synthesis, not exhaustive enumeration. A status at a named intermediate position is not a settled outcome. Example: "Explain a subject and include timestamps" => {"kind":"answer-timestamp","locator":null,"connectsBeyondPosition":false,"asksForSettledOutcome":false,"asksForExhaustiveCoverage":false}. Example: "Who won the match?" => {"kind":"none","locator":null,"connectsBeyondPosition":false,"asksForSettledOutcome":true,"asksForExhaustiveCoverage":false}. Example: "What happens at 14:00?" => {"kind":"input-location","locator":"14:00","connectsBeyondPosition":false,"asksForSettledOutcome":false,"asksForExhaustiveCoverage":false}. Example: "Who is at 14:00 and where else do they appear?" => {"kind":"input-location","locator":"14:00","connectsBeyondPosition":true,"asksForSettledOutcome":false,"asksForExhaustiveCoverage":false}. Example: "Give a complete timeline of every visual and spoken event from start to finish" => {"kind":"none","locator":null,"connectsBeyondPosition":false,"asksForSettledOutcome":false,"asksForExhaustiveCoverage":true}. Example: "Give the full story from beginning to conclusion" => {"kind":"none","locator":null,"connectsBeyondPosition":false,"asksForSettledOutcome":false,"asksForExhaustiveCoverage":false} unless the user asks specifically about the beginning or conclusion. Interpret every language without assuming a source genre.',
|
|
502
509
|
prompt: input.question,
|
|
503
510
|
});
|
|
504
511
|
const serialized = text.match(/\{[\s\S]*\}/)?.[0];
|
|
@@ -507,10 +514,19 @@ async function userSpecifiedSourcePosition(input: {
|
|
|
507
514
|
kind?: unknown;
|
|
508
515
|
locator?: unknown;
|
|
509
516
|
connectsBeyondPosition?: unknown;
|
|
517
|
+
asksForSettledOutcome?: unknown;
|
|
518
|
+
asksForExhaustiveCoverage?: unknown;
|
|
510
519
|
};
|
|
511
520
|
const connectsBeyondPosition = parsed.connectsBeyondPosition === true;
|
|
521
|
+
const asksForSettledOutcome = parsed.asksForSettledOutcome === true;
|
|
522
|
+
const asksForExhaustiveCoverage = parsed.asksForExhaustiveCoverage === true;
|
|
512
523
|
if (parsed.kind === 'answer-timestamp' || parsed.kind === 'none') {
|
|
513
|
-
return {
|
|
524
|
+
return {
|
|
525
|
+
hasPosition: false,
|
|
526
|
+
connectsBeyondPosition,
|
|
527
|
+
asksForSettledOutcome,
|
|
528
|
+
asksForExhaustiveCoverage,
|
|
529
|
+
};
|
|
514
530
|
}
|
|
515
531
|
if (parsed.kind !== 'input-location' || typeof parsed.locator !== 'string') return undefined;
|
|
516
532
|
// The quote need only occur in the user text. Its semantic role was
|
|
@@ -518,6 +534,8 @@ async function userSpecifiedSourcePosition(input: {
|
|
|
518
534
|
return {
|
|
519
535
|
hasPosition: input.question.normalize('NFKC').includes(parsed.locator.normalize('NFKC')),
|
|
520
536
|
connectsBeyondPosition,
|
|
537
|
+
asksForSettledOutcome,
|
|
538
|
+
asksForExhaustiveCoverage,
|
|
521
539
|
};
|
|
522
540
|
} catch {
|
|
523
541
|
return undefined;
|
|
@@ -535,7 +553,7 @@ function parseVideoInvestigationDirective(value: string): VideoInvestigationDire
|
|
|
535
553
|
const scope = ['focused', 'temporal', 'source'].includes(String(parsed.scope))
|
|
536
554
|
? (parsed.scope as VideoInvestigationDirective['scope'])
|
|
537
555
|
: 'focused';
|
|
538
|
-
const goal = ['answer', 'compare', 'trace', 'enumerate', 'synthesize'].includes(
|
|
556
|
+
const goal = ['answer', 'compare', 'trace', 'resolve', 'enumerate', 'synthesize'].includes(
|
|
539
557
|
String(parsed.goal),
|
|
540
558
|
)
|
|
541
559
|
? (parsed.goal as VideoInvestigationDirective['goal'])
|
|
@@ -1206,6 +1224,9 @@ ${fieldLines}`;
|
|
|
1206
1224
|
userText,
|
|
1207
1225
|
);
|
|
1208
1226
|
const imagePreviewFollowUp = isImagePreviewFollowUp(userText, reusableEvidence);
|
|
1227
|
+
const directImagePreviewInput = imagePreviewFollowUp
|
|
1228
|
+
? imagePreviewInput(reusableEvidence)
|
|
1229
|
+
: undefined;
|
|
1209
1230
|
const tabularFollowUp = isTabularFollowUp(userText, reusableEvidence);
|
|
1210
1231
|
const dataExportFormat = requestedDataExportFormat(userText);
|
|
1211
1232
|
const exportNeedsFreshData = dataExportNeedsFreshData(userText);
|
|
@@ -1328,6 +1349,53 @@ ${fieldLines}`;
|
|
|
1328
1349
|
delete (builtInTools as any).searchKnowledgeBase;
|
|
1329
1350
|
}
|
|
1330
1351
|
|
|
1352
|
+
// A direct "show it" follow-up already has a source-scoped image reference.
|
|
1353
|
+
// Render it on the host instead of repeating retrieval and a model round trip.
|
|
1354
|
+
// PDF pages are rendered fresh from the original source, not a stale derivative.
|
|
1355
|
+
if (directImagePreviewInput && builtInTools.presentMedia) {
|
|
1356
|
+
const previewCallId = `preview-${crypto.randomUUID()}`;
|
|
1357
|
+
const stream = createUIMessageStream({
|
|
1358
|
+
originalMessages: messages,
|
|
1359
|
+
execute: async ({ writer }) => {
|
|
1360
|
+
writer.write({ type: 'start' });
|
|
1361
|
+
writer.write({
|
|
1362
|
+
type: 'tool-input-available',
|
|
1363
|
+
toolCallId: previewCallId,
|
|
1364
|
+
toolName: 'presentMedia',
|
|
1365
|
+
input: directImagePreviewInput,
|
|
1366
|
+
});
|
|
1367
|
+
let previewOutput: any;
|
|
1368
|
+
try {
|
|
1369
|
+
previewOutput = await (builtInTools.presentMedia as any).execute(
|
|
1370
|
+
directImagePreviewInput,
|
|
1371
|
+
{ toolCallId: previewCallId },
|
|
1372
|
+
);
|
|
1373
|
+
} catch (error) {
|
|
1374
|
+
previewOutput = {
|
|
1375
|
+
success: false,
|
|
1376
|
+
error: error instanceof Error ? error.message : 'The source image could not be opened.',
|
|
1377
|
+
};
|
|
1378
|
+
}
|
|
1379
|
+
writer.write({
|
|
1380
|
+
type: 'tool-output-available',
|
|
1381
|
+
toolCallId: previewCallId,
|
|
1382
|
+
output: previewOutput,
|
|
1383
|
+
});
|
|
1384
|
+
const answerId = `answer-${crypto.randomUUID()}`;
|
|
1385
|
+
writer.write({ type: 'text-start', id: answerId });
|
|
1386
|
+
writer.write({
|
|
1387
|
+
type: 'text-delta',
|
|
1388
|
+
id: answerId,
|
|
1389
|
+
delta: previewOutput?.success
|
|
1390
|
+
? 'Here is the requested source image.'
|
|
1391
|
+
: previewOutput?.error || 'The source image could not be opened.',
|
|
1392
|
+
});
|
|
1393
|
+
writer.write({ type: 'text-end', id: answerId });
|
|
1394
|
+
},
|
|
1395
|
+
});
|
|
1396
|
+
return createUIMessageStreamResponse({ stream });
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1331
1399
|
// Remote MCP connections are opt-in per workspace and their tool names are
|
|
1332
1400
|
// namespaced by connection. This lets Local Chat use the same connections as
|
|
1333
1401
|
// Agents without risking collisions with Larkup's built-in tools.
|
|
@@ -1362,6 +1430,7 @@ ${fieldLines}`;
|
|
|
1362
1430
|
let preloadedEvidence: unknown;
|
|
1363
1431
|
let preloadedVideoEvidence = false;
|
|
1364
1432
|
let preloadedEvidenceQueryAttempted = false;
|
|
1433
|
+
let preloadedVideoInvestigation: VideoInvestigationDirective | undefined;
|
|
1365
1434
|
const preflightToolCallId = `knowledge-${crypto.randomUUID()}`;
|
|
1366
1435
|
|
|
1367
1436
|
if (directDataExport && dataExportFormat && builtInTools.createDataExport) {
|
|
@@ -1700,26 +1769,8 @@ ${fieldLines}`;
|
|
|
1700
1769
|
...plannerInput,
|
|
1701
1770
|
question: resolvedMediaQuestion,
|
|
1702
1771
|
});
|
|
1703
|
-
const
|
|
1704
|
-
|
|
1705
|
-
// `trace` is already the planner's language-neutral declaration
|
|
1706
|
-
// that the answer connects moments. Preserve its supplied source
|
|
1707
|
-
// boundary and give the evidence capability its visibility record
|
|
1708
|
-
// set even if the auxiliary binary classifier was inconclusive.
|
|
1709
|
-
const tracesBeyondPosition =
|
|
1710
|
-
connectsBeyondPosition || plannedInvestigation?.goal === 'trace';
|
|
1711
|
-
const investigation =
|
|
1712
|
-
hasUserSpecifiedPosition === false && plannedInvestigation?.timeRange
|
|
1713
|
-
? { scope: 'source' as const, goal: 'synthesize' as const }
|
|
1714
|
-
: tracesBeyondPosition && plannedInvestigation?.timeRange
|
|
1715
|
-
? {
|
|
1716
|
-
...plannedInvestigation,
|
|
1717
|
-
scope: 'temporal' as const,
|
|
1718
|
-
goal: 'trace' as const,
|
|
1719
|
-
evidence: ['visual'] as const,
|
|
1720
|
-
recordSet: 'observed' as const,
|
|
1721
|
-
}
|
|
1722
|
-
: plannedInvestigation;
|
|
1772
|
+
const investigation = reconcileVideoInvestigation(plannedInvestigation, sourcePosition);
|
|
1773
|
+
preloadedVideoInvestigation = investigation;
|
|
1723
1774
|
const evidenceInput = {
|
|
1724
1775
|
mediaAssetId,
|
|
1725
1776
|
query: resolvedMediaQuestion,
|
|
@@ -1776,10 +1827,15 @@ ${fieldLines}`;
|
|
|
1776
1827
|
const deterministicAnswer =
|
|
1777
1828
|
formatExhaustiveMediaAnswer(preloadedEvidence, userText) ??
|
|
1778
1829
|
(useVisualDeterministicAnswer
|
|
1779
|
-
? (
|
|
1830
|
+
? ((preloadedVideoInvestigation
|
|
1831
|
+
? formatOutcomeMediaAnswer(preloadedEvidence, preloadedVideoInvestigation)
|
|
1832
|
+
: undefined) ??
|
|
1833
|
+
formatLocatedObservedSubjectAnswer(preloadedEvidence) ??
|
|
1780
1834
|
collectQuestionMatchedDirectClaims(preloadedEvidence, userText)[0] ??
|
|
1781
1835
|
formatDirectObservationAnswer(preloadedEvidence, userText) ??
|
|
1782
|
-
|
|
1836
|
+
(preloadedVideoInvestigation?.goal === 'resolve'
|
|
1837
|
+
? undefined
|
|
1838
|
+
: formatObservedAppearanceAnswer(preloadedEvidence, preloadedVideoInvestigation)))
|
|
1783
1839
|
: undefined);
|
|
1784
1840
|
if (deterministicAnswer) {
|
|
1785
1841
|
const answerId = `answer-${crypto.randomUUID()}`;
|
|
@@ -1328,7 +1328,7 @@ export async function getChatTools(context: {
|
|
|
1328
1328
|
investigation: z
|
|
1329
1329
|
.object({
|
|
1330
1330
|
scope: z.enum(['focused', 'temporal', 'source']),
|
|
1331
|
-
goal: z.enum(['answer', 'compare', 'trace', 'enumerate', 'synthesize']),
|
|
1331
|
+
goal: z.enum(['answer', 'compare', 'trace', 'resolve', 'enumerate', 'synthesize']),
|
|
1332
1332
|
evidence: z
|
|
1333
1333
|
.array(z.enum(['speech', 'visible-text', 'visual', 'computed']))
|
|
1334
1334
|
.max(4)
|
|
@@ -2306,7 +2306,7 @@ export async function getChatTools(context: {
|
|
|
2306
2306
|
question: z.string().describe('The full user question, including any temporal conditions.'),
|
|
2307
2307
|
investigation: z.object({
|
|
2308
2308
|
scope: z.enum(['focused', 'temporal', 'source']),
|
|
2309
|
-
goal: z.enum(['answer', 'compare', 'trace', 'enumerate', 'synthesize']),
|
|
2309
|
+
goal: z.enum(['answer', 'compare', 'trace', 'resolve', 'enumerate', 'synthesize']),
|
|
2310
2310
|
evidence: z
|
|
2311
2311
|
.array(z.enum(['speech', 'visible-text', 'visual', 'computed']))
|
|
2312
2312
|
.max(4)
|
|
@@ -26,7 +26,8 @@ export type ExactGroundedAnswer = {
|
|
|
26
26
|
};
|
|
27
27
|
|
|
28
28
|
export type ReusableImageEvidence = {
|
|
29
|
-
imageUrl
|
|
29
|
+
imageUrl?: string;
|
|
30
|
+
documentId?: string;
|
|
30
31
|
pageNumber?: number;
|
|
31
32
|
index?: number;
|
|
32
33
|
title?: string;
|
|
@@ -208,13 +209,18 @@ export function findImmediateExactGroundedAnswer(
|
|
|
208
209
|
return undefined;
|
|
209
210
|
}
|
|
210
211
|
|
|
211
|
-
function imageEvidence(
|
|
212
|
+
function imageEvidence(
|
|
213
|
+
value: unknown,
|
|
214
|
+
title?: string,
|
|
215
|
+
documentId?: string,
|
|
216
|
+
): ReusableImageEvidence[] {
|
|
212
217
|
if (!Array.isArray(value)) return [];
|
|
213
218
|
return value.flatMap((image: any) => {
|
|
214
219
|
if (!image || typeof image.imageUrl !== 'string' || !image.imageUrl) return [];
|
|
215
220
|
return [
|
|
216
221
|
{
|
|
217
222
|
imageUrl: image.imageUrl,
|
|
223
|
+
documentId,
|
|
218
224
|
pageNumber: typeof image.pageNumber === 'number' ? image.pageNumber : undefined,
|
|
219
225
|
index: typeof image.index === 'number' ? image.index : undefined,
|
|
220
226
|
title,
|
|
@@ -349,6 +355,28 @@ export function extractConversationEvidence(
|
|
|
349
355
|
return { sources: [], images: [], mediaAssetIds: [], tabular: table };
|
|
350
356
|
}
|
|
351
357
|
|
|
358
|
+
const inspectedPdfPages = parts
|
|
359
|
+
.filter((part) => part.name === 'inspectPdfPages')
|
|
360
|
+
.flatMap((part) => {
|
|
361
|
+
const input = unwrapJson(part.input) as { documentId?: unknown } | undefined;
|
|
362
|
+
const output = unwrapJson(part.output) as
|
|
363
|
+
| { success?: unknown; pages?: Array<{ pageNumber?: unknown; imageUrl?: unknown }> }
|
|
364
|
+
| undefined;
|
|
365
|
+
if (typeof input?.documentId !== 'string' || output?.success !== true) return [];
|
|
366
|
+
return (output.pages ?? []).flatMap((page) => {
|
|
367
|
+
if (typeof page.pageNumber !== 'number' || !Number.isInteger(page.pageNumber)) return [];
|
|
368
|
+
return [
|
|
369
|
+
{
|
|
370
|
+
documentId: input.documentId as string,
|
|
371
|
+
pageNumber: page.pageNumber,
|
|
372
|
+
...(typeof page.imageUrl === 'string' && page.imageUrl
|
|
373
|
+
? { imageUrl: page.imageUrl }
|
|
374
|
+
: {}),
|
|
375
|
+
} satisfies ReusableImageEvidence,
|
|
376
|
+
];
|
|
377
|
+
});
|
|
378
|
+
});
|
|
379
|
+
|
|
352
380
|
const result = parts
|
|
353
381
|
.filter((part) => part.name === 'searchKnowledgeBase')
|
|
354
382
|
.map(
|
|
@@ -369,14 +397,17 @@ export function extractConversationEvidence(
|
|
|
369
397
|
const hits = Array.isArray(result?.hits) ? result.hits : [];
|
|
370
398
|
|
|
371
399
|
const images = new Map<string, ReusableImageEvidence>();
|
|
400
|
+
for (const image of inspectedPdfPages) {
|
|
401
|
+
images.set(`${image.documentId}:${image.pageNumber}`, image);
|
|
402
|
+
}
|
|
372
403
|
for (const hit of hits.slice(0, 4) as any[]) {
|
|
373
404
|
if (!hit || typeof hit !== 'object') continue;
|
|
374
405
|
const title = typeof hit.title === 'string' ? hit.title : undefined;
|
|
375
406
|
for (const image of [
|
|
376
|
-
...imageEvidence(hit.images, title),
|
|
377
|
-
...imageEvidence(hit.metadata?.images, title),
|
|
407
|
+
...imageEvidence(hit.images, title, hit.documentId),
|
|
408
|
+
...imageEvidence(hit.metadata?.images, title, hit.documentId),
|
|
378
409
|
]) {
|
|
379
|
-
images.set(image.imageUrl
|
|
410
|
+
images.set(image.imageUrl ?? `${image.documentId}:${image.pageNumber}`, image);
|
|
380
411
|
}
|
|
381
412
|
}
|
|
382
413
|
const sources = hits.slice(0, 2).flatMap((hit: any) => {
|
|
@@ -500,11 +531,23 @@ export function isTabularFollowUp(text: string, evidence: ConversationEvidence):
|
|
|
500
531
|
|
|
501
532
|
export function isImagePreviewFollowUp(text: string, evidence: ConversationEvidence): boolean {
|
|
502
533
|
if (evidence.images.length === 0) return false;
|
|
503
|
-
return /\b(?:show|preview|display|open|view|render)\b[\s\S]{0,40}\b(?:image|picture|diagram|page|figure|it)\b|\b(?:image|picture|diagram|figure)\s+preview\b/i.test(
|
|
534
|
+
return /\b(?:show|preview|display|open|view|render|need|give|provide|want)\b[\s\S]{0,40}\b(?:image|picture|diagram|page|figure|visual|it)\b|\b(?:image|picture|diagram|figure|visual)\s+(?:preview|view)\b/i.test(
|
|
504
535
|
text,
|
|
505
536
|
);
|
|
506
537
|
}
|
|
507
538
|
|
|
539
|
+
/** Reuse the exact prior source, preferring a fresh PDF-page render over a stale derivative. */
|
|
540
|
+
export function imagePreviewInput(
|
|
541
|
+
evidence: ConversationEvidence,
|
|
542
|
+
): { documentId: string; pageNumber: number } | { imageUrl: string } | undefined {
|
|
543
|
+
const image = evidence.images[0];
|
|
544
|
+
if (!image) return undefined;
|
|
545
|
+
if (image.documentId && image.pageNumber) {
|
|
546
|
+
return { documentId: image.documentId, pageNumber: image.pageNumber };
|
|
547
|
+
}
|
|
548
|
+
return image.imageUrl ? { imageUrl: image.imageUrl } : undefined;
|
|
549
|
+
}
|
|
550
|
+
|
|
508
551
|
/**
|
|
509
552
|
* A source is selected by structured evidence from the immediately preceding
|
|
510
553
|
* turn. Keep it until a later source-selection event replaces it; never infer
|
|
@@ -441,7 +441,18 @@ export function formatLocatedObservedSubjectAnswer(value: unknown): string | und
|
|
|
441
441
|
* asking a prose model to reinterpret explicit discrete moments as one
|
|
442
442
|
* continuous appearance or as an absence claim.
|
|
443
443
|
*/
|
|
444
|
-
export function formatObservedAppearanceAnswer(
|
|
444
|
+
export function formatObservedAppearanceAnswer(
|
|
445
|
+
value: unknown,
|
|
446
|
+
investigation?: { goal?: unknown; recordSet?: unknown },
|
|
447
|
+
): string | undefined {
|
|
448
|
+
// A visibility ledger answers recurrence/presence questions. It is context,
|
|
449
|
+
// not a substitute for another requested visual fact such as clothing,
|
|
450
|
+
// colour, action, count, or identity. The typed planner directive is the
|
|
451
|
+
// language-neutral boundary: only an explicit observed trace may render the
|
|
452
|
+
// ledger as the final answer.
|
|
453
|
+
if (investigation?.goal !== 'trace' || investigation.recordSet !== 'observed') {
|
|
454
|
+
return undefined;
|
|
455
|
+
}
|
|
445
456
|
type Appearance = { startSecs: number; endSecs: number };
|
|
446
457
|
type Subject = { identity: string; basis: string; appearances: Appearance[] };
|
|
447
458
|
const subjects = new Map<string, Subject>();
|
|
@@ -544,7 +555,7 @@ export function formatOutcomeMediaAnswer(
|
|
|
544
555
|
value: unknown,
|
|
545
556
|
investigation: VideoInvestigationDirective,
|
|
546
557
|
): string | undefined {
|
|
547
|
-
if (investigation.goal !== 'trace') return undefined;
|
|
558
|
+
if (investigation.goal !== 'trace' && investigation.goal !== 'resolve') return undefined;
|
|
548
559
|
type Reading = { at: number; text: string };
|
|
549
560
|
const readings: Reading[] = [];
|
|
550
561
|
const visit = (candidate: unknown) => {
|
|
@@ -575,6 +586,16 @@ export function formatOutcomeMediaAnswer(
|
|
|
575
586
|
};
|
|
576
587
|
visit(value);
|
|
577
588
|
|
|
589
|
+
const settledWinner = readings
|
|
590
|
+
.sort((left, right) => left.at - right.at)
|
|
591
|
+
.flatMap((reading) => {
|
|
592
|
+
const match = reading.text.match(
|
|
593
|
+
/^Reconciled event:\s*(?:.*?\b(?:conclusion|final)\b.*?)\bwith\s+(.+?)\s+as\s+(?:the\s+)?winner\s*$/iu,
|
|
594
|
+
);
|
|
595
|
+
return match?.[1]?.trim() ? [{ ...reading, winner: match[1].trim() }] : [];
|
|
596
|
+
})
|
|
597
|
+
.at(-1);
|
|
598
|
+
|
|
578
599
|
const cleanLabel = (label: string) =>
|
|
579
600
|
label
|
|
580
601
|
.replace(/^.*?(?:(?:final\s+)?(?:result|score)|النتيجة(?:\s+النهائية)?)\s*:\s*/iu, '')
|
|
@@ -607,6 +628,22 @@ export function formatOutcomeMediaAnswer(
|
|
|
607
628
|
: [];
|
|
608
629
|
});
|
|
609
630
|
});
|
|
631
|
+
const sideScorePattern =
|
|
632
|
+
/^\s*(.+?\p{L})\s+(\d+(?:[.,]\d+)?)\s*[||]\s*(.+?\p{L})\s+(\d+(?:[.,]\d+)?)\s*$/u;
|
|
633
|
+
const sideScoreCandidates = readings
|
|
634
|
+
.sort((left, right) => left.at - right.at)
|
|
635
|
+
.flatMap((reading) => {
|
|
636
|
+
const match = cleanLabel(reading.text).match(sideScorePattern);
|
|
637
|
+
if (!match) return [];
|
|
638
|
+
const left = { label: cleanLabel(match[1]), value: Number(match[2].replace(',', '.')) };
|
|
639
|
+
const right = { label: cleanLabel(match[3]), value: Number(match[4].replace(',', '.')) };
|
|
640
|
+
return left.label &&
|
|
641
|
+
right.label &&
|
|
642
|
+
Number.isFinite(left.value) &&
|
|
643
|
+
Number.isFinite(right.value)
|
|
644
|
+
? [{ ...reading, pairs: [left, right] }]
|
|
645
|
+
: [];
|
|
646
|
+
});
|
|
610
647
|
const pairPattern =
|
|
611
648
|
/([^=,;،؛\n]{2,100}?)(?:\s*=\s*|\s*:\s*|\s+)(-?\d+(?:[.,]\d+)?)(?=$|[,;،؛])/gu;
|
|
612
649
|
const candidates = readings
|
|
@@ -622,7 +659,14 @@ export function formatOutcomeMediaAnswer(
|
|
|
622
659
|
};
|
|
623
660
|
})
|
|
624
661
|
.filter((reading) => reading.pairs.length === 2);
|
|
625
|
-
const settled = scorelineCandidates.at(-1) ?? candidates.at(-1);
|
|
662
|
+
const settled = scorelineCandidates.at(-1) ?? sideScoreCandidates.at(-1) ?? candidates.at(-1);
|
|
663
|
+
if (settledWinner) {
|
|
664
|
+
if (!settled) return `${settledWinner.winner} won.`;
|
|
665
|
+
const values = settled.pairs.map((pair) => pair.value).sort((left, right) => right - left);
|
|
666
|
+
if (values[0] !== values[1]) {
|
|
667
|
+
return `${settledWinner.winner} won, ${values[0]}–${values[1]}.`;
|
|
668
|
+
}
|
|
669
|
+
}
|
|
626
670
|
if (!settled) return undefined;
|
|
627
671
|
const [left, right] = settled.pairs;
|
|
628
672
|
if (left.value === right.value) {
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { VideoInvestigationDirective } from '@larkup/core/video-knowledge/query-planner';
|
|
2
|
+
|
|
3
|
+
export interface VideoQuestionSignals {
|
|
4
|
+
hasPosition?: boolean;
|
|
5
|
+
connectsBeyondPosition?: boolean;
|
|
6
|
+
asksForSettledOutcome?: boolean;
|
|
7
|
+
asksForExhaustiveCoverage?: boolean;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Reconcile the planner directive with the independent question classifier.
|
|
12
|
+
* The classifier is deliberately language-neutral and has final authority over
|
|
13
|
+
* outcome requests, because a broad planner can otherwise mistake “who won?”
|
|
14
|
+
* for source synthesis and send the answer through appearance rendering.
|
|
15
|
+
*/
|
|
16
|
+
export function reconcileVideoInvestigation(
|
|
17
|
+
planned: VideoInvestigationDirective | undefined,
|
|
18
|
+
signals: VideoQuestionSignals | undefined,
|
|
19
|
+
): VideoInvestigationDirective | undefined {
|
|
20
|
+
if (signals?.asksForSettledOutcome) {
|
|
21
|
+
return { scope: 'temporal', goal: 'resolve' };
|
|
22
|
+
}
|
|
23
|
+
if (signals?.hasPosition === false && signals.asksForExhaustiveCoverage) {
|
|
24
|
+
return {
|
|
25
|
+
scope: 'source',
|
|
26
|
+
goal: 'enumerate',
|
|
27
|
+
...(planned?.evidence?.length ? { evidence: planned.evidence } : {}),
|
|
28
|
+
recordSet: 'all',
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
if (signals?.hasPosition === false && planned?.timeRange) {
|
|
32
|
+
return { scope: 'source', goal: 'synthesize' };
|
|
33
|
+
}
|
|
34
|
+
if ((signals?.connectsBeyondPosition || planned?.goal === 'trace') && planned?.timeRange) {
|
|
35
|
+
return {
|
|
36
|
+
...planned,
|
|
37
|
+
scope: 'temporal',
|
|
38
|
+
goal: 'trace',
|
|
39
|
+
evidence: ['visual'],
|
|
40
|
+
recordSet: 'observed',
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
return planned;
|
|
44
|
+
}
|