@larkup/workspace-server 0.2.9 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-types/app/api/chat/route.d.ts.map +1 -1
- package/dist-types/lib/chat/conversation-memory.d.ts +9 -1
- package/dist-types/lib/chat/conversation-memory.d.ts.map +1 -1
- package/dist-types/lib/chat/tool-context.d.ts +4 -1
- package/dist-types/lib/chat/tool-context.d.ts.map +1 -1
- package/dist-types/lib/chat/video-query-planning.d.ts +1 -0
- package/dist-types/lib/chat/video-query-planning.d.ts.map +1 -1
- package/package.json +4 -3
- package/src/app/api/chat/route.ts +64 -3
- package/src/lib/chat/conversation-memory.ts +49 -6
- package/src/lib/chat/tool-context.ts +12 -1
- package/src/lib/chat/video-query-planning.ts +9 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"route.d.ts","sourceRoot":"","sources":["../../../../src/app/api/chat/route.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"route.d.ts","sourceRoot":"","sources":["../../../../src/app/api/chat/route.ts"],"names":[],"mappings":"AAqHA,eAAO,MAAM,WAAW,MAAM,CAAC;AAiuB/B,wBAAsB,IAAI,CAAC,GAAG,EAAE,OAAO,qBA69CtC"}
|
|
@@ -9,7 +9,8 @@ export type ExactGroundedAnswer = {
|
|
|
9
9
|
sourceScopeFingerprint: string;
|
|
10
10
|
};
|
|
11
11
|
export type ReusableImageEvidence = {
|
|
12
|
-
imageUrl
|
|
12
|
+
imageUrl?: string;
|
|
13
|
+
documentId?: string;
|
|
13
14
|
pageNumber?: number;
|
|
14
15
|
index?: number;
|
|
15
16
|
title?: string;
|
|
@@ -57,6 +58,13 @@ export declare function extractConversationEvidence(messages: readonly ChatMessa
|
|
|
57
58
|
*/
|
|
58
59
|
export declare function isTabularFollowUp(text: string, evidence: ConversationEvidence): boolean;
|
|
59
60
|
export declare function isImagePreviewFollowUp(text: string, evidence: ConversationEvidence): boolean;
|
|
61
|
+
/** Reuse the exact prior source, preferring a fresh PDF-page render over a stale derivative. */
|
|
62
|
+
export declare function imagePreviewInput(evidence: ConversationEvidence): {
|
|
63
|
+
documentId: string;
|
|
64
|
+
pageNumber: number;
|
|
65
|
+
} | {
|
|
66
|
+
imageUrl: string;
|
|
67
|
+
} | undefined;
|
|
60
68
|
/**
|
|
61
69
|
* A source is selected by structured evidence from the immediately preceding
|
|
62
70
|
* turn. Keep it until a later source-selection event replaces it; never infer
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"conversation-memory.d.ts","sourceRoot":"","sources":["../../../src/lib/chat/conversation-memory.ts"],"names":[],"mappings":"AAeA,KAAK,WAAW,GAAG;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,sBAAsB,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"conversation-memory.d.ts","sourceRoot":"","sources":["../../../src/lib/chat/conversation-memory.ts"],"names":[],"mappings":"AAeA,KAAK,WAAW,GAAG;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,sBAAsB,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACrC,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,0FAA0F;AAC1F,wBAAgB,iCAAiC,CAC/C,QAAQ,EAAE,uBAAuB,GAAG,SAAS,EAC7C,mBAAmB,EAAE,WAAW,CAAC,MAAM,CAAC,GACvC,OAAO,CAET;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClD,MAAM,EAAE,qBAAqB,EAAE,CAAC;IAChC,yFAAyF;IACzF,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,6FAA6F;IAC7F,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,+EAA+E;IAC/E,OAAO,CAAC,EAAE,uBAAuB,CAAC;CACnC,CAAC;AAoFF;;;;GAIG;AACH,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,SAAS,WAAW,EAAE,EAChC,eAAe,EAAE,MAAM,GACtB,mBAAmB,GAAG,SAAS,CA0DjC;AA4ED,gFAAgF;AAChF,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,SAAS,WAAW,EAAE,GAC/B,uBAAuB,GAAG,SAAS,CAUrC;AAED,sFAAsF;AACtF,wBAAgB,iCAAiC,CAC/C,MAAM,EAAE,OAAO,EAAE,EACjB,OAAO,EAAE,MAAM,EAAE,GAChB,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAuBhC;AAYD;;;GAGG;AACH,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,SAAS,WAAW,EAAE,GAC/B,oBAAoB,CAmItB;AAmCD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,oBAAoB,GAAG,OAAO,CAavF;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,oBAAoB,GAAG,OAAO,CAK5F;AAED,gGAAgG;AAChG,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,oBAAoB,GAC7B;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAO/E;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,oBAAoB,GAAG,OAAO,CAE/F;AAUD;;;;GAIG;AACH,wBAAgB,iCAAiC,CAC/C,eAAe,EAAE,MAAM,EACvB,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAepB;AAED;;;;GAIG;AACH,wBAAgB,mCAAmC,CACjD,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,oBAAoB,GAC7B,MAAM,GAAG,SAAS,CAyBpB;AAED,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,oBAAoB,GAAG,MAAM,CAoCjF"}
|
|
@@ -38,7 +38,10 @@ export declare function formatLocatedObservedSubjectAnswer(value: unknown): stri
|
|
|
38
38
|
* asking a prose model to reinterpret explicit discrete moments as one
|
|
39
39
|
* continuous appearance or as an absence claim.
|
|
40
40
|
*/
|
|
41
|
-
export declare function formatObservedAppearanceAnswer(value: unknown
|
|
41
|
+
export declare function formatObservedAppearanceAnswer(value: unknown, investigation?: {
|
|
42
|
+
goal?: unknown;
|
|
43
|
+
recordSet?: unknown;
|
|
44
|
+
}): string | undefined;
|
|
42
45
|
/** Render a final two-sided numeric outcome without waiting on a slow prose model. */
|
|
43
46
|
export declare function formatOutcomeMediaAnswer(value: unknown, investigation: VideoInvestigationDirective): string | undefined;
|
|
44
47
|
/** Use the strongest bounded source reading directly when it already settled the question. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-context.d.ts","sourceRoot":"","sources":["../../../src/lib/chat/tool-context.ts"],"names":[],"mappings":"AAUA;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAmBpE;AAED,0EAA0E;AAC1E,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAgB/D;AAED,kFAAkF;AAClF,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAmBxE;AAmBD,oEAAoE;AACpE,wBAAgB,kCAAkC,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAuB7F;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CA+D1E;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CAyDrE;AAED;;;;;GAKG;AACH,wBAAgB,kCAAkC,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CA4KrF;AAED;;;;;;GAMG;AACH,wBAAgB,8BAA8B,
|
|
1
|
+
{"version":3,"file":"tool-context.d.ts","sourceRoot":"","sources":["../../../src/lib/chat/tool-context.ts"],"names":[],"mappings":"AAUA;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAmBpE;AAED,0EAA0E;AAC1E,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAgB/D;AAED,kFAAkF;AAClF,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAmBxE;AAmBD,oEAAoE;AACpE,wBAAgB,kCAAkC,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAuB7F;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CA+D1E;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CAyDrE;AAED;;;;;GAKG;AACH,wBAAgB,kCAAkC,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CA4KrF;AAED;;;;;;GAMG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,OAAO,EACd,aAAa,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GACtD,MAAM,GAAG,SAAS,CAwGpB;AAED,sFAAsF;AACtF,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,2BAA2B,GACzC,MAAM,GAAG,SAAS,CAyHpB;AAED,8FAA8F;AAC9F,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,SAAK,GAAG,MAAM,GAAG,SAAS,CAqE/F;AAED,8EAA8E;AAC9E,wBAAgB,+BAA+B,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CA2BvE;AAED,sGAAsG;AACtG,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CA0D/F;AAYD;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CA+IjG;AAED,gGAAgG;AAChG,wBAAgB,2BAA2B,CAAC,YAAY,EAAE,MAAM,mCAwC/D;AAkLD;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,SAAS,GAAG,EAAE,GAAG,GAAG,EAAE,CAapE;AAED,uFAAuF;AACvF,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,SAAS,GAAG,EAAE,GAAG,GAAG,EAAE,CAiB1E;AACD,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,4CAA4C,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"video-query-planning.d.ts","sourceRoot":"","sources":["../../../src/lib/chat/video-query-planning.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,4CAA4C,CAAC;AAE9F,MAAM,WAAW,oBAAoB;IACnC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,qBAAqB,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"video-query-planning.d.ts","sourceRoot":"","sources":["../../../src/lib/chat/video-query-planning.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,4CAA4C,CAAC;AAE9F,MAAM,WAAW,oBAAoB;IACnC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB,CAAC,EAAE,OAAO,CAAC;CACrC;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,OAAO,EAAE,2BAA2B,GAAG,SAAS,EAChD,OAAO,EAAE,oBAAoB,GAAG,SAAS,GACxC,2BAA2B,GAAG,SAAS,CAyBzC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@larkup/workspace-server",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.14",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"files": [
|
|
6
6
|
"src",
|
|
@@ -319,14 +319,15 @@
|
|
|
319
319
|
"@ai-sdk/openai-compatible": "3.0.66",
|
|
320
320
|
"@larkup/core": "^0.8.2",
|
|
321
321
|
"@larkup/tool-doc-editor": "^0.3.2",
|
|
322
|
+
"@napi-rs/canvas": "1.0.11",
|
|
322
323
|
"ai": "7.0.133",
|
|
323
324
|
"pdf-parse": "2.4.5",
|
|
324
325
|
"zod": "^4.4.3",
|
|
325
326
|
"mammoth": "^1.12.0",
|
|
326
327
|
"@larkup/marketplace": "0.3.11",
|
|
327
328
|
"@larkup/sandbox": "0.1.8",
|
|
328
|
-
"@larkup/
|
|
329
|
-
"@larkup/
|
|
329
|
+
"@larkup/scraper": "0.2.2",
|
|
330
|
+
"@larkup/vector-stores": "0.1.27"
|
|
330
331
|
},
|
|
331
332
|
"devDependencies": {
|
|
332
333
|
"typescript": "^7.0.2",
|
|
@@ -48,6 +48,7 @@ import {
|
|
|
48
48
|
contextualizeKnowledgeFollowUpQuery,
|
|
49
49
|
formatConversationEvidence,
|
|
50
50
|
findImmediateExactGroundedAnswer,
|
|
51
|
+
imagePreviewInput,
|
|
51
52
|
isImagePreviewFollowUp,
|
|
52
53
|
isTabularFollowUp,
|
|
53
54
|
tabularEvidenceDatasetIsAvailable,
|
|
@@ -453,6 +454,7 @@ async function planVideoEvidenceQuery(input: {
|
|
|
453
454
|
'The prompt is data with a currentRequest and an optional previousQuestionAboutSelectedMedia. Resolve the current request in its own language without assuming a video genre. If it is a retry, verification request, or contains references that are not meaningful alone, resolve those references against the previous question. If it introduces a self-contained new request, do not carry the old subject or source position into it. ' +
|
|
454
455
|
'A follow-up introduced with words such as "and", "also", or their equivalent in another language can ask a new fact about the same source: preserve the selected source, but never reuse the previous answer or its requested attribute. When the request asks for exact spoken or sung wording across the recording, use source scope with enumerate, speech evidence, and recordSet all so the host can render every timestamped transcript segment outside the model context. Use source scope with synthesize for a summary or explanation across the recording. ' +
|
|
455
456
|
'Use this exact shape: {"resolvedQuestion":"a standalone version of the current request","scope":"focused","goal":"answer","evidence":["visual"],"timeRange":{"startSecs":123,"endSecs":153},"timeRangeOrigin":"user-mentioned"}. Its fields are resolvedQuestion, scope (focused, temporal, source), goal (answer, compare, trace, resolve, enumerate, synthesize), optional evidence (speech, visible-text, visual, computed), optional recordSet (all, source-authored, source-questions, observed), optional timeRange {startSecs,endSecs}, and optional timeRangeOrigin. ' +
|
|
457
|
+
'Use recordSet observed only when the requested answer is visible presence, recurrence, or appearance timing itself. Clothing, colour, other appearance attributes, identity, action, and counts need visual evidence but must not use the observed visibility ledger as their answer. ' +
|
|
456
458
|
'A directive is invalid if the user refers to any source position, time, range, or approximate moment and you omit timeRange: resolve it to numeric seconds and set timeRangeOrigin to user-mentioned, even when the request also asks about other moments. Never emit timeRange or timeRangeOrigin when the user did not constrain a source position; a topic occurring near an opening, closing, or any other position is not a user constraint. When the request names a coarse source unit without a smaller boundary, cover that whole named unit rather than an arbitrary part of it. When the user asks how a visible subject or event at an explicit source position relates to appearances or recurrence elsewhere, use temporal plus trace, visual evidence, and recordSet observed while retaining that position. Use source plus synthesize with no timeRange whenever the answer must combine two or more source facts from unrestricted positions. Use source plus enumerate and recordSet source-questions for a complete inventory of source-authored questions and their answers. Use temporal for a relationship or development across moments; use focused only for one local moment or one self-contained fact. ' +
|
|
457
459
|
'Use temporal plus resolve when the requested fact is the settled or final outcome of a process, contest, decision, vote, test, or other sequence; this includes asking who won or what the final result was in any language. Do not confuse an entity mentioned in that request with a request to inventory its visual appearances. ' +
|
|
458
460
|
'Do not answer the question or add fields.',
|
|
@@ -490,6 +492,7 @@ async function userSpecifiedSourcePosition(input: {
|
|
|
490
492
|
hasPosition: boolean;
|
|
491
493
|
connectsBeyondPosition: boolean;
|
|
492
494
|
asksForSettledOutcome: boolean;
|
|
495
|
+
asksForExhaustiveCoverage: boolean;
|
|
493
496
|
}
|
|
494
497
|
| undefined
|
|
495
498
|
> {
|
|
@@ -502,7 +505,7 @@ async function userSpecifiedSourcePosition(input: {
|
|
|
502
505
|
abortSignal: AbortSignal.timeout(8_000),
|
|
503
506
|
providerOptions: input.providerOptions,
|
|
504
507
|
system:
|
|
505
|
-
'Return exactly one JSON object and no prose: {"kind":"input-location"|"answer-timestamp"|"none","locator":string|null,"connectsBeyondPosition":boolean,"asksForSettledOutcome":boolean}. Use input-location only when the user explicitly constrains where in the provided source to look; locator must be the exact corresponding substring from the user request. Use answer-timestamp when the user asks for timestamps in the answer but does not specify where to look; locator is null. Use none when neither is present. Set connectsBeyondPosition true only when the request explicitly relates the thing at its named source position to another point or portion of the source, such as recurrence, reappearance, a later/earlier occurrence, change, or comparison; otherwise false. Set asksForSettledOutcome true only when the request asks for the final or settled result of a process, contest, decision, vote, test, or other sequence, including who won in any language; it overrides a broad summary classification. A status at a named intermediate position is not a settled outcome. Example: "Explain a subject and include timestamps" => {"kind":"answer-timestamp","locator":null,"connectsBeyondPosition":false,"asksForSettledOutcome":false}. Example: "Who won the match?" => {"kind":"none","locator":null,"connectsBeyondPosition":false,"asksForSettledOutcome":true}. Example: "What happens at 14:00?" => {"kind":"input-location","locator":"14:00","connectsBeyondPosition":false,"asksForSettledOutcome":false}. Example: "Who is at 14:00 and where else do they appear?" => {"kind":"input-location","locator":"14:00","connectsBeyondPosition":true,"asksForSettledOutcome":false}. Example: "Give the full story from beginning to conclusion" => {"kind":"none","locator":null,"connectsBeyondPosition":false,"asksForSettledOutcome":false} unless the user asks specifically about the beginning or conclusion. Interpret every language without assuming a source genre.',
|
|
508
|
+
'Return exactly one JSON object and no prose: {"kind":"input-location"|"answer-timestamp"|"none","locator":string|null,"connectsBeyondPosition":boolean,"asksForSettledOutcome":boolean,"asksForExhaustiveCoverage":boolean}. Use input-location only when the user explicitly constrains where in the provided source to look; locator must be the exact corresponding substring from the user request. Use answer-timestamp when the user asks for timestamps in the answer but does not specify where to look; locator is null. Use none when neither is present. Set connectsBeyondPosition true only when the request explicitly relates the thing at its named source position to another point or portion of the source, such as recurrence, reappearance, a later/earlier occurrence, change, or comparison; otherwise false. Set asksForSettledOutcome true only when the request asks for the final or settled result of a process, contest, decision, vote, test, or other sequence, including who won in any language; it overrides a broad summary classification. Set asksForExhaustiveCoverage true only when the user explicitly asks to enumerate all or every relevant record across the complete recording, such as a complete start-to-finish timeline, every spoken statement, or every visual and spoken event. A full story, overview, or summary is source-wide synthesis, not exhaustive enumeration. A status at a named intermediate position is not a settled outcome. Example: "Explain a subject and include timestamps" => {"kind":"answer-timestamp","locator":null,"connectsBeyondPosition":false,"asksForSettledOutcome":false,"asksForExhaustiveCoverage":false}. Example: "Who won the match?" => {"kind":"none","locator":null,"connectsBeyondPosition":false,"asksForSettledOutcome":true,"asksForExhaustiveCoverage":false}. Example: "What happens at 14:00?" => {"kind":"input-location","locator":"14:00","connectsBeyondPosition":false,"asksForSettledOutcome":false,"asksForExhaustiveCoverage":false}. Example: "Who is at 14:00 and where else do they appear?" => {"kind":"input-location","locator":"14:00","connectsBeyondPosition":true,"asksForSettledOutcome":false,"asksForExhaustiveCoverage":false}. Example: "Give a complete timeline of every visual and spoken event from start to finish" => {"kind":"none","locator":null,"connectsBeyondPosition":false,"asksForSettledOutcome":false,"asksForExhaustiveCoverage":true}. Example: "Give the full story from beginning to conclusion" => {"kind":"none","locator":null,"connectsBeyondPosition":false,"asksForSettledOutcome":false,"asksForExhaustiveCoverage":false} unless the user asks specifically about the beginning or conclusion. Interpret every language without assuming a source genre.',
|
|
506
509
|
prompt: input.question,
|
|
507
510
|
});
|
|
508
511
|
const serialized = text.match(/\{[\s\S]*\}/)?.[0];
|
|
@@ -512,11 +515,18 @@ async function userSpecifiedSourcePosition(input: {
|
|
|
512
515
|
locator?: unknown;
|
|
513
516
|
connectsBeyondPosition?: unknown;
|
|
514
517
|
asksForSettledOutcome?: unknown;
|
|
518
|
+
asksForExhaustiveCoverage?: unknown;
|
|
515
519
|
};
|
|
516
520
|
const connectsBeyondPosition = parsed.connectsBeyondPosition === true;
|
|
517
521
|
const asksForSettledOutcome = parsed.asksForSettledOutcome === true;
|
|
522
|
+
const asksForExhaustiveCoverage = parsed.asksForExhaustiveCoverage === true;
|
|
518
523
|
if (parsed.kind === 'answer-timestamp' || parsed.kind === 'none') {
|
|
519
|
-
return {
|
|
524
|
+
return {
|
|
525
|
+
hasPosition: false,
|
|
526
|
+
connectsBeyondPosition,
|
|
527
|
+
asksForSettledOutcome,
|
|
528
|
+
asksForExhaustiveCoverage,
|
|
529
|
+
};
|
|
520
530
|
}
|
|
521
531
|
if (parsed.kind !== 'input-location' || typeof parsed.locator !== 'string') return undefined;
|
|
522
532
|
// The quote need only occur in the user text. Its semantic role was
|
|
@@ -525,6 +535,7 @@ async function userSpecifiedSourcePosition(input: {
|
|
|
525
535
|
hasPosition: input.question.normalize('NFKC').includes(parsed.locator.normalize('NFKC')),
|
|
526
536
|
connectsBeyondPosition,
|
|
527
537
|
asksForSettledOutcome,
|
|
538
|
+
asksForExhaustiveCoverage,
|
|
528
539
|
};
|
|
529
540
|
} catch {
|
|
530
541
|
return undefined;
|
|
@@ -1213,6 +1224,9 @@ ${fieldLines}`;
|
|
|
1213
1224
|
userText,
|
|
1214
1225
|
);
|
|
1215
1226
|
const imagePreviewFollowUp = isImagePreviewFollowUp(userText, reusableEvidence);
|
|
1227
|
+
const directImagePreviewInput = imagePreviewFollowUp
|
|
1228
|
+
? imagePreviewInput(reusableEvidence)
|
|
1229
|
+
: undefined;
|
|
1216
1230
|
const tabularFollowUp = isTabularFollowUp(userText, reusableEvidence);
|
|
1217
1231
|
const dataExportFormat = requestedDataExportFormat(userText);
|
|
1218
1232
|
const exportNeedsFreshData = dataExportNeedsFreshData(userText);
|
|
@@ -1335,6 +1349,53 @@ ${fieldLines}`;
|
|
|
1335
1349
|
delete (builtInTools as any).searchKnowledgeBase;
|
|
1336
1350
|
}
|
|
1337
1351
|
|
|
1352
|
+
// A direct "show it" follow-up already has a source-scoped image reference.
|
|
1353
|
+
// Render it on the host instead of repeating retrieval and a model round trip.
|
|
1354
|
+
// PDF pages are rendered fresh from the original source, not a stale derivative.
|
|
1355
|
+
if (directImagePreviewInput && builtInTools.presentMedia) {
|
|
1356
|
+
const previewCallId = `preview-${crypto.randomUUID()}`;
|
|
1357
|
+
const stream = createUIMessageStream({
|
|
1358
|
+
originalMessages: messages,
|
|
1359
|
+
execute: async ({ writer }) => {
|
|
1360
|
+
writer.write({ type: 'start' });
|
|
1361
|
+
writer.write({
|
|
1362
|
+
type: 'tool-input-available',
|
|
1363
|
+
toolCallId: previewCallId,
|
|
1364
|
+
toolName: 'presentMedia',
|
|
1365
|
+
input: directImagePreviewInput,
|
|
1366
|
+
});
|
|
1367
|
+
let previewOutput: any;
|
|
1368
|
+
try {
|
|
1369
|
+
previewOutput = await (builtInTools.presentMedia as any).execute(
|
|
1370
|
+
directImagePreviewInput,
|
|
1371
|
+
{ toolCallId: previewCallId },
|
|
1372
|
+
);
|
|
1373
|
+
} catch (error) {
|
|
1374
|
+
previewOutput = {
|
|
1375
|
+
success: false,
|
|
1376
|
+
error: error instanceof Error ? error.message : 'The source image could not be opened.',
|
|
1377
|
+
};
|
|
1378
|
+
}
|
|
1379
|
+
writer.write({
|
|
1380
|
+
type: 'tool-output-available',
|
|
1381
|
+
toolCallId: previewCallId,
|
|
1382
|
+
output: previewOutput,
|
|
1383
|
+
});
|
|
1384
|
+
const answerId = `answer-${crypto.randomUUID()}`;
|
|
1385
|
+
writer.write({ type: 'text-start', id: answerId });
|
|
1386
|
+
writer.write({
|
|
1387
|
+
type: 'text-delta',
|
|
1388
|
+
id: answerId,
|
|
1389
|
+
delta: previewOutput?.success
|
|
1390
|
+
? 'Here is the requested source image.'
|
|
1391
|
+
: previewOutput?.error || 'The source image could not be opened.',
|
|
1392
|
+
});
|
|
1393
|
+
writer.write({ type: 'text-end', id: answerId });
|
|
1394
|
+
},
|
|
1395
|
+
});
|
|
1396
|
+
return createUIMessageStreamResponse({ stream });
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1338
1399
|
// Remote MCP connections are opt-in per workspace and their tool names are
|
|
1339
1400
|
// namespaced by connection. This lets Local Chat use the same connections as
|
|
1340
1401
|
// Agents without risking collisions with Larkup's built-in tools.
|
|
@@ -1774,7 +1835,7 @@ ${fieldLines}`;
|
|
|
1774
1835
|
formatDirectObservationAnswer(preloadedEvidence, userText) ??
|
|
1775
1836
|
(preloadedVideoInvestigation?.goal === 'resolve'
|
|
1776
1837
|
? undefined
|
|
1777
|
-
: formatObservedAppearanceAnswer(preloadedEvidence)))
|
|
1838
|
+
: formatObservedAppearanceAnswer(preloadedEvidence, preloadedVideoInvestigation)))
|
|
1778
1839
|
: undefined);
|
|
1779
1840
|
if (deterministicAnswer) {
|
|
1780
1841
|
const answerId = `answer-${crypto.randomUUID()}`;
|
|
@@ -26,7 +26,8 @@ export type ExactGroundedAnswer = {
|
|
|
26
26
|
};
|
|
27
27
|
|
|
28
28
|
export type ReusableImageEvidence = {
|
|
29
|
-
imageUrl
|
|
29
|
+
imageUrl?: string;
|
|
30
|
+
documentId?: string;
|
|
30
31
|
pageNumber?: number;
|
|
31
32
|
index?: number;
|
|
32
33
|
title?: string;
|
|
@@ -208,13 +209,18 @@ export function findImmediateExactGroundedAnswer(
|
|
|
208
209
|
return undefined;
|
|
209
210
|
}
|
|
210
211
|
|
|
211
|
-
function imageEvidence(
|
|
212
|
+
function imageEvidence(
|
|
213
|
+
value: unknown,
|
|
214
|
+
title?: string,
|
|
215
|
+
documentId?: string,
|
|
216
|
+
): ReusableImageEvidence[] {
|
|
212
217
|
if (!Array.isArray(value)) return [];
|
|
213
218
|
return value.flatMap((image: any) => {
|
|
214
219
|
if (!image || typeof image.imageUrl !== 'string' || !image.imageUrl) return [];
|
|
215
220
|
return [
|
|
216
221
|
{
|
|
217
222
|
imageUrl: image.imageUrl,
|
|
223
|
+
documentId,
|
|
218
224
|
pageNumber: typeof image.pageNumber === 'number' ? image.pageNumber : undefined,
|
|
219
225
|
index: typeof image.index === 'number' ? image.index : undefined,
|
|
220
226
|
title,
|
|
@@ -349,6 +355,28 @@ export function extractConversationEvidence(
|
|
|
349
355
|
return { sources: [], images: [], mediaAssetIds: [], tabular: table };
|
|
350
356
|
}
|
|
351
357
|
|
|
358
|
+
const inspectedPdfPages = parts
|
|
359
|
+
.filter((part) => part.name === 'inspectPdfPages')
|
|
360
|
+
.flatMap((part) => {
|
|
361
|
+
const input = unwrapJson(part.input) as { documentId?: unknown } | undefined;
|
|
362
|
+
const output = unwrapJson(part.output) as
|
|
363
|
+
| { success?: unknown; pages?: Array<{ pageNumber?: unknown; imageUrl?: unknown }> }
|
|
364
|
+
| undefined;
|
|
365
|
+
if (typeof input?.documentId !== 'string' || output?.success !== true) return [];
|
|
366
|
+
return (output.pages ?? []).flatMap((page) => {
|
|
367
|
+
if (typeof page.pageNumber !== 'number' || !Number.isInteger(page.pageNumber)) return [];
|
|
368
|
+
return [
|
|
369
|
+
{
|
|
370
|
+
documentId: input.documentId as string,
|
|
371
|
+
pageNumber: page.pageNumber,
|
|
372
|
+
...(typeof page.imageUrl === 'string' && page.imageUrl
|
|
373
|
+
? { imageUrl: page.imageUrl }
|
|
374
|
+
: {}),
|
|
375
|
+
} satisfies ReusableImageEvidence,
|
|
376
|
+
];
|
|
377
|
+
});
|
|
378
|
+
});
|
|
379
|
+
|
|
352
380
|
const result = parts
|
|
353
381
|
.filter((part) => part.name === 'searchKnowledgeBase')
|
|
354
382
|
.map(
|
|
@@ -369,14 +397,17 @@ export function extractConversationEvidence(
|
|
|
369
397
|
const hits = Array.isArray(result?.hits) ? result.hits : [];
|
|
370
398
|
|
|
371
399
|
const images = new Map<string, ReusableImageEvidence>();
|
|
400
|
+
for (const image of inspectedPdfPages) {
|
|
401
|
+
images.set(`${image.documentId}:${image.pageNumber}`, image);
|
|
402
|
+
}
|
|
372
403
|
for (const hit of hits.slice(0, 4) as any[]) {
|
|
373
404
|
if (!hit || typeof hit !== 'object') continue;
|
|
374
405
|
const title = typeof hit.title === 'string' ? hit.title : undefined;
|
|
375
406
|
for (const image of [
|
|
376
|
-
...imageEvidence(hit.images, title),
|
|
377
|
-
...imageEvidence(hit.metadata?.images, title),
|
|
407
|
+
...imageEvidence(hit.images, title, hit.documentId),
|
|
408
|
+
...imageEvidence(hit.metadata?.images, title, hit.documentId),
|
|
378
409
|
]) {
|
|
379
|
-
images.set(image.imageUrl
|
|
410
|
+
images.set(image.imageUrl ?? `${image.documentId}:${image.pageNumber}`, image);
|
|
380
411
|
}
|
|
381
412
|
}
|
|
382
413
|
const sources = hits.slice(0, 2).flatMap((hit: any) => {
|
|
@@ -500,11 +531,23 @@ export function isTabularFollowUp(text: string, evidence: ConversationEvidence):
|
|
|
500
531
|
|
|
501
532
|
export function isImagePreviewFollowUp(text: string, evidence: ConversationEvidence): boolean {
|
|
502
533
|
if (evidence.images.length === 0) return false;
|
|
503
|
-
return /\b(?:show|preview|display|open|view|render)\b[\s\S]{0,40}\b(?:image|picture|diagram|page|figure|it)\b|\b(?:image|picture|diagram|figure)\s+preview\b/i.test(
|
|
534
|
+
return /\b(?:show|preview|display|open|view|render|need|give|provide|want)\b[\s\S]{0,40}\b(?:image|picture|diagram|page|figure|visual|it)\b|\b(?:image|picture|diagram|figure|visual)\s+(?:preview|view)\b/i.test(
|
|
504
535
|
text,
|
|
505
536
|
);
|
|
506
537
|
}
|
|
507
538
|
|
|
539
|
+
/** Reuse the exact prior source, preferring a fresh PDF-page render over a stale derivative. */
|
|
540
|
+
export function imagePreviewInput(
|
|
541
|
+
evidence: ConversationEvidence,
|
|
542
|
+
): { documentId: string; pageNumber: number } | { imageUrl: string } | undefined {
|
|
543
|
+
const image = evidence.images[0];
|
|
544
|
+
if (!image) return undefined;
|
|
545
|
+
if (image.documentId && image.pageNumber) {
|
|
546
|
+
return { documentId: image.documentId, pageNumber: image.pageNumber };
|
|
547
|
+
}
|
|
548
|
+
return image.imageUrl ? { imageUrl: image.imageUrl } : undefined;
|
|
549
|
+
}
|
|
550
|
+
|
|
508
551
|
/**
|
|
509
552
|
* A source is selected by structured evidence from the immediately preceding
|
|
510
553
|
* turn. Keep it until a later source-selection event replaces it; never infer
|
|
@@ -441,7 +441,18 @@ export function formatLocatedObservedSubjectAnswer(value: unknown): string | und
|
|
|
441
441
|
* asking a prose model to reinterpret explicit discrete moments as one
|
|
442
442
|
* continuous appearance or as an absence claim.
|
|
443
443
|
*/
|
|
444
|
-
export function formatObservedAppearanceAnswer(
|
|
444
|
+
export function formatObservedAppearanceAnswer(
|
|
445
|
+
value: unknown,
|
|
446
|
+
investigation?: { goal?: unknown; recordSet?: unknown },
|
|
447
|
+
): string | undefined {
|
|
448
|
+
// A visibility ledger answers recurrence/presence questions. It is context,
|
|
449
|
+
// not a substitute for another requested visual fact such as clothing,
|
|
450
|
+
// colour, action, count, or identity. The typed planner directive is the
|
|
451
|
+
// language-neutral boundary: only an explicit observed trace may render the
|
|
452
|
+
// ledger as the final answer.
|
|
453
|
+
if (investigation?.goal !== 'trace' || investigation.recordSet !== 'observed') {
|
|
454
|
+
return undefined;
|
|
455
|
+
}
|
|
445
456
|
type Appearance = { startSecs: number; endSecs: number };
|
|
446
457
|
type Subject = { identity: string; basis: string; appearances: Appearance[] };
|
|
447
458
|
const subjects = new Map<string, Subject>();
|
|
@@ -4,6 +4,7 @@ export interface VideoQuestionSignals {
|
|
|
4
4
|
hasPosition?: boolean;
|
|
5
5
|
connectsBeyondPosition?: boolean;
|
|
6
6
|
asksForSettledOutcome?: boolean;
|
|
7
|
+
asksForExhaustiveCoverage?: boolean;
|
|
7
8
|
}
|
|
8
9
|
|
|
9
10
|
/**
|
|
@@ -19,6 +20,14 @@ export function reconcileVideoInvestigation(
|
|
|
19
20
|
if (signals?.asksForSettledOutcome) {
|
|
20
21
|
return { scope: 'temporal', goal: 'resolve' };
|
|
21
22
|
}
|
|
23
|
+
if (signals?.hasPosition === false && signals.asksForExhaustiveCoverage) {
|
|
24
|
+
return {
|
|
25
|
+
scope: 'source',
|
|
26
|
+
goal: 'enumerate',
|
|
27
|
+
...(planned?.evidence?.length ? { evidence: planned.evidence } : {}),
|
|
28
|
+
recordSet: 'all',
|
|
29
|
+
};
|
|
30
|
+
}
|
|
22
31
|
if (signals?.hasPosition === false && planned?.timeRange) {
|
|
23
32
|
return { scope: 'source', goal: 'synthesize' };
|
|
24
33
|
}
|