@qvac/core 0.10.3 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -1
- package/dist-lib/lib/assistant/history.d.ts +2 -1
- package/dist-lib/lib/assistant/history.js +32 -5
- package/dist-lib/lib/assistant/message.d.ts +20 -0
- package/dist-lib/lib/assistant/message.js +36 -0
- package/dist-lib/lib/assistant/pdf-budget.d.ts +3 -0
- package/dist-lib/lib/assistant/pdf-budget.js +13 -0
- package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
- package/dist-lib/lib/assistant/video-budget.js +29 -0
- package/dist-lib/lib/assistant-tree.d.ts +1 -0
- package/dist-lib/lib/assistant-tree.js +9 -2
- package/dist-lib/lib/assistant.d.ts +26 -3
- package/dist-lib/lib/assistant.js +471 -47
- package/dist-lib/lib/core/chat-fork.js +1 -0
- package/dist-lib/lib/core/chat-search.js +1 -1
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +5 -1
- package/dist-lib/lib/core/departures.js +6 -0
- package/dist-lib/lib/core/document-attachment.d.ts +7 -0
- package/dist-lib/lib/core/document-attachment.js +9 -0
- package/dist-lib/lib/core/files/index.js +4 -4
- package/dist-lib/lib/core/files/records/chunks.js +21 -0
- package/dist-lib/lib/core/files/records/purge.js +4 -3
- package/dist-lib/lib/core/files/records/source.js +3 -5
- package/dist-lib/lib/core/files/upload.js +3 -2
- package/dist-lib/lib/core/identity.d.ts +2 -0
- package/dist-lib/lib/core/identity.js +19 -0
- package/dist-lib/lib/core/pdf/rasterize.d.ts +0 -1
- package/dist-lib/lib/core/pdf/rasterize.js +0 -1
- package/dist-lib/lib/core/video/attachment.d.ts +1 -0
- package/dist-lib/lib/core/video/attachment.js +7 -0
- package/dist-lib/lib/core/video/digest.d.ts +16 -0
- package/dist-lib/lib/core/video/digest.js +57 -0
- package/dist-lib/lib/core/video/frames.d.ts +32 -0
- package/dist-lib/lib/core/video/frames.js +224 -0
- package/dist-lib/lib/core.d.ts +1 -0
- package/dist-lib/lib/core.js +44 -7
- package/dist-lib/lib/credential-pusher.d.ts +6 -0
- package/dist-lib/lib/credential-pusher.js +76 -59
- package/dist-lib/lib/errors.d.ts +8 -0
- package/dist-lib/lib/errors.js +8 -0
- package/dist-lib/lib/oauth/flow-core.js +2 -0
- package/dist-lib/lib/oauth/loopback.d.ts +1 -0
- package/dist-lib/lib/oauth/loopback.js +31 -1
- package/dist-lib/lib/oauth/register.js +33 -18
- package/dist-lib/lib/oauth/request.d.ts +10 -0
- package/dist-lib/lib/oauth/request.js +22 -0
- package/dist-lib/lib/oauth/token-exchange-core.js +29 -32
- package/dist-lib/lib/oauth/types.d.ts +3 -0
- package/dist-lib/lib/oauth.d.ts +2 -1
- package/dist-lib/lib/oauth.js +1 -0
- package/dist-lib/lib/run-notices.d.ts +1 -0
- package/dist-lib/lib/run-notices.js +2 -1
- package/dist-lib/lib/run-replay.d.ts +8 -2
- package/dist-lib/lib/run-replay.js +29 -11
- package/dist-lib/lib/task.js +1 -1
- package/dist-lib/schema/engine.js +4 -2
- package/dist-lib/schema/indexes.js +2 -2
- package/dist-lib/schema/qvac.js +4 -2
- package/dist-lib/spec/hrpc/messages.js +16 -6
- package/dist-lib/spec/hyperschema/index.js +16 -6
- package/dist-lib/spec/hyperschema/schema.json +10 -0
- package/dist-lib/spec/hyperschema/types.d.ts +2 -0
- package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
- package/package.json +7 -4
- package/tui/app.mjs +34 -1
- package/tui/lib/render.mjs +5 -0
- package/tui/main.mjs +8 -0
|
@@ -22,10 +22,10 @@ import { buildTitlePrompt, deriveChatTitle, isDefaultChatTitle, parseTitleRespon
|
|
|
22
22
|
import { matchApprovalRule } from './core/approval-rules.js';
|
|
23
23
|
import { isAudioAttachment } from './core/audio-attachment.js';
|
|
24
24
|
import { attachTurnDocumentIds, formatRagContext, chooseRagNotice, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, createRagSourceTagFilter, citedByPhraseOverlap, formatFullDocumentContext, isFullTextHit, isTruncatedFullTextHit, sourcesFromHits, WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from './core/files/index.js';
|
|
25
|
-
import { annotateDocumentAttachments, documentUnreadNotice, documentUnreadReason } from './core/document-attachment.js';
|
|
25
|
+
import { annotateDocumentAttachments, documentUnreadNotice, documentUnreadReason, pdfPagesCutNotice, pdfPagesSkippedNotice } from './core/document-attachment.js';
|
|
26
26
|
import { imageCacheKey, imageCacheName, sweepImageCache } from './core/image/cache-sweep.js';
|
|
27
27
|
import { IMAGE_MIMES, normalizeImageForModel } from './core/image/normalize.js';
|
|
28
|
-
import {
|
|
28
|
+
import { createPdfRasterizer } from './core/pdf/rasterize.js';
|
|
29
29
|
import { recordBackends, summarizeRuns } from './diagnostics.js';
|
|
30
30
|
import { resolveFsGrants } from './fs-drive-bridge.js';
|
|
31
31
|
import { InferenceCallStore } from './inference-calls.js';
|
|
@@ -44,14 +44,19 @@ import { isLifecycleTeardown, knownAbortOutcome, RUN_ABORT_REASONS, RUN_FAILURE_
|
|
|
44
44
|
import { unref } from './unref.js';
|
|
45
45
|
import { encodeAgentSkillReadiness } from './skill-readiness.js';
|
|
46
46
|
import { harnessClearKvCache, harnessDeathReason, harnessDeleteModel, harnessDied, harnessListModels, harnessPreviewModelSource, harnessSkillCatalog, harnessSkillManagement, harnessDetectLanguage, harnessSpeak, harnessSpeakStream, harnessTools, harnessTranscribe, heldLoadModel, droppedHeldLoadModel, imageDownloadProgress, isHarnessEvent, modelCatalogChanged, toolReportEntry } from './assistant/harness.js';
|
|
47
|
-
import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders } from './assistant/message.js';
|
|
47
|
+
import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders, restoreSentContent, hiddenVideoNotice, lentVideoNotice, unreadVideoNotice, unsyncedVideoNotice, VIDEO_NO_AUDIO, VIDEO_SILENT, videoAnnotation, videoTranscriptBlock } from './assistant/message.js';
|
|
48
|
+
import { frameTimes, probeVideo, sampleFrames } from './core/video/frames.js';
|
|
49
|
+
import { packVideoDigest, parseVideoDigestHeader, unpackVideoDigest, VIDEO_DIGEST_MIME, videoDigestFileName } from './core/video/digest.js';
|
|
50
|
+
import { VIDEO_FRAME_EDGE, videoFrameCount } from './assistant/video-budget.js';
|
|
48
51
|
import { drainBody, raceInactivity } from './assistant/fetch-body.js';
|
|
52
|
+
import { pdfPageBudget } from './assistant/pdf-budget.js';
|
|
53
|
+
import { effectiveCtxSize } from './api/effective-ctx-size.js';
|
|
49
54
|
import { GENERIC_ACKNOWLEDGEMENTS, HISTORY_TYPES, SKILL_TOOL, TERMINAL_STATES, couldNameSenders, historyFromChunks, inputAvailableToRun, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds } from './assistant/history.js';
|
|
50
55
|
import { errorMessage, isRecord } from './guards.js';
|
|
51
56
|
import { CoreError, isCoreError } from './errors.js';
|
|
52
57
|
export { harnessClearKvCache, harnessDeleteModel, harnessListModels, harnessPreviewModelSource, harnessSkillCatalog, harnessSkillManagement, harnessDetectLanguage, harnessSpeak, harnessSpeakStream, harnessTools, harnessTranscribe } from './assistant/harness.js';
|
|
53
58
|
export {} from './assistant/skill-encode.js';
|
|
54
|
-
export { annotateUnreadableImages, unheardAudioNotice, unreadableImagesNotice, DOCUMENT_ONLY_USER_PROMPT, IMAGE_AND_DOCUMENT_USER_PROMPT, IMAGE_ONLY_USER_PROMPT, nameSenders } from './assistant/message.js';
|
|
59
|
+
export { annotateUnreadableImages, unheardAudioNotice, unreadableImagesNotice, DOCUMENT_ONLY_USER_PROMPT, IMAGE_AND_DOCUMENT_USER_PROMPT, IMAGE_ONLY_USER_PROMPT, nameSenders, restoreSentContent } from './assistant/message.js';
|
|
55
60
|
export { couldNameSenders, historyFromChunks, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds } from './assistant/history.js';
|
|
56
61
|
const NO_SENDERS = new Map();
|
|
57
62
|
const DEFAULT_PROFILE = 'mid';
|
|
@@ -59,6 +64,7 @@ const QvacState = local.getEnum('@local/qvac-state');
|
|
|
59
64
|
const FINAL_FLUSH_MS = 10_000;
|
|
60
65
|
const UPLOAD_FRAME_BYTES = 64 * 1024;
|
|
61
66
|
const RUN_IDLE_TIMEOUT_MS = 15 * 60_000;
|
|
67
|
+
const MODEL_RELOAD_DELAY_MS = 2_500;
|
|
62
68
|
function idleWatch(controller, ms) {
|
|
63
69
|
let last = 0;
|
|
64
70
|
let parked = 0;
|
|
@@ -106,6 +112,20 @@ const RunOccupancySchema = z.object({
|
|
|
106
112
|
export const CAPTURE_INFERENCE_CALLS_KEY = 'inspector.captureInferenceCalls';
|
|
107
113
|
export const CAPTURE_ON = 'on';
|
|
108
114
|
export const CAPTURE_OFF = 'off';
|
|
115
|
+
function pdfCut(doc, from) {
|
|
116
|
+
const pageCount = doc.pageCount();
|
|
117
|
+
for (let page = from; page < pageCount; page++) {
|
|
118
|
+
try {
|
|
119
|
+
const flags = doc.pageFlags(page);
|
|
120
|
+
if (flags.hasImage || !flags.hasText) {
|
|
121
|
+
return { lastPage: page, pageCount };
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return undefined;
|
|
128
|
+
}
|
|
109
129
|
function attachPages(turn, files) {
|
|
110
130
|
if (!files.length)
|
|
111
131
|
return;
|
|
@@ -114,6 +134,24 @@ function attachPages(turn, files) {
|
|
|
114
134
|
function noBytes() {
|
|
115
135
|
return Promise.resolve(null);
|
|
116
136
|
}
|
|
137
|
+
function videoFrameCacheName(video, count, frame) {
|
|
138
|
+
return imageCacheName({ blobId: video.blobId, fileName: `frames.${count}.f${frame}.jpg` });
|
|
139
|
+
}
|
|
140
|
+
function videoHeaderCacheName(digestBlobId) {
|
|
141
|
+
return imageCacheName({ blobId: digestBlobId, fileName: 'frames.json' });
|
|
142
|
+
}
|
|
143
|
+
function videoAudioBlock(fileName, { audio, transcript }) {
|
|
144
|
+
if (audio === 'none') {
|
|
145
|
+
return VIDEO_NO_AUDIO;
|
|
146
|
+
}
|
|
147
|
+
if (audio === 'silent') {
|
|
148
|
+
return VIDEO_SILENT;
|
|
149
|
+
}
|
|
150
|
+
if (audio === 'speech' && transcript) {
|
|
151
|
+
return videoTranscriptBlock(transcript);
|
|
152
|
+
}
|
|
153
|
+
return unheardAudioNotice(`The audio of the attached video "${fileName}"`, 'transcription failed');
|
|
154
|
+
}
|
|
117
155
|
function pdfPageCacheName(pdf, page) {
|
|
118
156
|
return imageCacheName({ blobId: pdf.blobId, fileName: `${pdf.fileName}.p${page}.jpg` });
|
|
119
157
|
}
|
|
@@ -146,6 +184,11 @@ function isSetUpHere(status) {
|
|
|
146
184
|
function sameNameList(left, right) {
|
|
147
185
|
return left.length === right.length && left.every((name, index) => name === right[index]);
|
|
148
186
|
}
|
|
187
|
+
function sameLoadSettings(a, b) {
|
|
188
|
+
return ((a.ctxSize ?? 0) === (b.ctxSize ?? 0) &&
|
|
189
|
+
(a.kvCacheK ?? '') === (b.kvCacheK ?? '') &&
|
|
190
|
+
(a.kvCacheV ?? '') === (b.kvCacheV ?? ''));
|
|
191
|
+
}
|
|
149
192
|
function withConfigDefaults(agent, config) {
|
|
150
193
|
if (agent.ctxSize || !config?.ctxSize)
|
|
151
194
|
return agent;
|
|
@@ -178,6 +221,8 @@ export class QvacAssistant extends ReadyResource {
|
|
|
178
221
|
_drives;
|
|
179
222
|
_blobReadTimeoutMs;
|
|
180
223
|
_runIdleTimeoutMs;
|
|
224
|
+
_modelReloadDelayMs;
|
|
225
|
+
_reloads = new Map();
|
|
181
226
|
_pdfRasterizer;
|
|
182
227
|
_warmAbort = new AbortController();
|
|
183
228
|
_unloading = new Map();
|
|
@@ -218,6 +263,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
218
263
|
_runReplay;
|
|
219
264
|
_lifecycle;
|
|
220
265
|
_audioTranscripts;
|
|
266
|
+
_videoProbes;
|
|
221
267
|
_pdfPages;
|
|
222
268
|
_chats;
|
|
223
269
|
_runImageCacheNames;
|
|
@@ -226,7 +272,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
226
272
|
_modelCatalog = new ModelCatalogRevisionHub();
|
|
227
273
|
_onHarnessStatus;
|
|
228
274
|
_onModelCatalogEvent;
|
|
229
|
-
constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
|
|
275
|
+
constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, modelReloadDelayMs = MODEL_RELOAD_DELAY_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
|
|
230
276
|
super();
|
|
231
277
|
this._engine = engine;
|
|
232
278
|
this._harness = harness;
|
|
@@ -242,6 +288,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
242
288
|
this._preloadCurrentAgentModel = preloadCurrentAgentModel;
|
|
243
289
|
this._blobReadTimeoutMs = blobReadTimeoutMs;
|
|
244
290
|
this._runIdleTimeoutMs = runIdleTimeoutMs;
|
|
291
|
+
this._modelReloadDelayMs = modelReloadDelayMs;
|
|
245
292
|
this._pdfRasterizer = pdfRasterizerOption ?? createPdfRasterizer(logger);
|
|
246
293
|
this._deviceId = null;
|
|
247
294
|
this._info = { toolNames: [], skills: [], supportsImages: false };
|
|
@@ -268,6 +315,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
268
315
|
this._runReplay = new RunReplayStore(runReplayDir ?? persistentDir('run-replay'));
|
|
269
316
|
this._lifecycle = new AbortController();
|
|
270
317
|
this._audioTranscripts = new Map();
|
|
318
|
+
this._videoProbes = new Map();
|
|
271
319
|
this._pdfPages = new Map();
|
|
272
320
|
this._chats = new Map();
|
|
273
321
|
this._runImageCacheNames = new Map();
|
|
@@ -731,6 +779,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
731
779
|
this._harness.off?.('status', this._onHarnessStatus);
|
|
732
780
|
this._harness.off?.('model', this._onModelCatalogEvent);
|
|
733
781
|
this._warmAbort.abort(RUNNER_CLOSED);
|
|
782
|
+
this._cancelModelReloads();
|
|
734
783
|
this._onlineBoot.destroy(new CoreError('ASSISTANT_CLOSING'));
|
|
735
784
|
for (const run of this._runs.values()) {
|
|
736
785
|
run.controller.abort(RUNNER_CLOSED);
|
|
@@ -769,6 +818,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
769
818
|
this._log?.info('assistant', 'suspending', { runs: this._runs.size });
|
|
770
819
|
this._parked.clear();
|
|
771
820
|
this._stopMeshIntake();
|
|
821
|
+
this._cancelModelReloads();
|
|
772
822
|
await this._downloads?.suspend();
|
|
773
823
|
this._meshList?.destroy();
|
|
774
824
|
this._accountFrames?.destroy();
|
|
@@ -851,6 +901,44 @@ export class QvacAssistant extends ReadyResource {
|
|
|
851
901
|
const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
|
|
852
902
|
return rows?.some((row) => row.name === model && row.isCached) === true;
|
|
853
903
|
}
|
|
904
|
+
_scheduleModelReload(agent) {
|
|
905
|
+
const deviceId = this._deviceId;
|
|
906
|
+
if (!this._harness.loadModel ||
|
|
907
|
+
!deviceId ||
|
|
908
|
+
!agent.modelName ||
|
|
909
|
+
!agent.providerDeviceId ||
|
|
910
|
+
!b4a.equals(agent.providerDeviceId, deviceId)) {
|
|
911
|
+
return;
|
|
912
|
+
}
|
|
913
|
+
const name = agent.modelName;
|
|
914
|
+
clearTimeout(this._reloads.get(name));
|
|
915
|
+
const timer = setTimeout(() => {
|
|
916
|
+
this._reloads.delete(name);
|
|
917
|
+
this._reloadLoadedModel(agent).catch((err) => this._log?.warn('models', 'model reload failed', { model: name, err }));
|
|
918
|
+
}, this._modelReloadDelayMs);
|
|
919
|
+
unref(timer);
|
|
920
|
+
this._reloads.set(name, timer);
|
|
921
|
+
}
|
|
922
|
+
async _reloadLoadedModel(agent) {
|
|
923
|
+
if (this.closing || this.suspended || !this._harness.loadModel)
|
|
924
|
+
return;
|
|
925
|
+
const model = runnableModelName(agent.modelName, await this._runnableHost());
|
|
926
|
+
const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
|
|
927
|
+
const busy = [...this._runs.values()].some((run) => run.executing && !run.borrowed && run.model === model);
|
|
928
|
+
if (busy || !rows?.some((row) => row.name === model && row.isLoaded)) {
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
931
|
+
const config = await this._engine.getConfig({}).catch(noop);
|
|
932
|
+
const knobs = withConfigDefaults(agent, config);
|
|
933
|
+
const { loaded, reason } = await this._harness.loadModel({ model, agent: knobs });
|
|
934
|
+
this._log?.info('models', 'model reload requested', { model, loaded, reason });
|
|
935
|
+
}
|
|
936
|
+
_cancelModelReloads() {
|
|
937
|
+
for (const timer of this._reloads.values()) {
|
|
938
|
+
clearTimeout(timer);
|
|
939
|
+
}
|
|
940
|
+
this._reloads.clear();
|
|
941
|
+
}
|
|
854
942
|
async _agentRunning(model, activeAgentId) {
|
|
855
943
|
const deviceId = this._requireDeviceId();
|
|
856
944
|
const meshId = this._accountMeshId;
|
|
@@ -1073,21 +1161,26 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1073
1161
|
.forget(chatId)
|
|
1074
1162
|
.catch((err) => this._log?.warn('assistant', 'inspector records not dropped', { err }));
|
|
1075
1163
|
}
|
|
1076
|
-
async
|
|
1164
|
+
async _followAgentEdits(meshId, frames) {
|
|
1077
1165
|
let known = null;
|
|
1078
1166
|
for await (const frame of frames) {
|
|
1079
1167
|
if (this.closing || this.suspended)
|
|
1080
1168
|
return;
|
|
1081
1169
|
const agents = frame.agents ?? [];
|
|
1082
1170
|
this._followHosted(meshId, agents);
|
|
1083
|
-
const next = new Map(agents.map((agent) => [agent.id, agentRekeyFingerprint(agent)]));
|
|
1084
1171
|
for (const agent of agents) {
|
|
1085
1172
|
const previous = known?.get(agent.id);
|
|
1086
|
-
if (previous
|
|
1173
|
+
if (!previous) {
|
|
1174
|
+
continue;
|
|
1175
|
+
}
|
|
1176
|
+
if (agentRekeyFingerprint(previous) !== agentRekeyFingerprint(agent)) {
|
|
1087
1177
|
void this._reclaimAgentChatCaches(meshId, agent);
|
|
1088
1178
|
}
|
|
1179
|
+
if (!sameLoadSettings(previous, agent)) {
|
|
1180
|
+
this._scheduleModelReload(agent);
|
|
1181
|
+
}
|
|
1089
1182
|
}
|
|
1090
|
-
known =
|
|
1183
|
+
known = new Map(agents.map((agent) => [agent.id, agent]));
|
|
1091
1184
|
}
|
|
1092
1185
|
}
|
|
1093
1186
|
async _reclaimAgentChatCaches(meshId, agent) {
|
|
@@ -1195,18 +1288,21 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1195
1288
|
}
|
|
1196
1289
|
async _sentRuns(chatId, holder) {
|
|
1197
1290
|
try {
|
|
1198
|
-
return
|
|
1291
|
+
return {
|
|
1292
|
+
sentRuns: await this._runReplay.load({ holder, chatId }),
|
|
1293
|
+
askedRuns: await this._runReplay.loadAsked({ holder, chatId })
|
|
1294
|
+
};
|
|
1199
1295
|
}
|
|
1200
1296
|
catch (err) {
|
|
1201
1297
|
this._log?.warn('assistant', 'run replay unreadable', { chatId, err });
|
|
1202
1298
|
await this._deleteChatKvCache(chatId, 'run replay unreadable', holder);
|
|
1203
|
-
return new Map();
|
|
1299
|
+
return { sentRuns: new Map(), askedRuns: new Map() };
|
|
1204
1300
|
}
|
|
1205
1301
|
}
|
|
1206
|
-
async _recordSentRun(chatId, { runId, turns, holder }) {
|
|
1302
|
+
async _recordSentRun(chatId, { runId, turns, asked, holder }) {
|
|
1207
1303
|
if (turns) {
|
|
1208
1304
|
try {
|
|
1209
|
-
await this._runReplay.save({ holder, chatId }, runId, turns);
|
|
1305
|
+
await this._runReplay.save({ holder, chatId }, runId, { messages: turns, asked });
|
|
1210
1306
|
return;
|
|
1211
1307
|
}
|
|
1212
1308
|
catch (err) {
|
|
@@ -1300,7 +1396,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1300
1396
|
const watches = [
|
|
1301
1397
|
this._streamWatch(`run-requests:${meshId}`, () => this._engine.runRequestsWatch({ meshId }), (frames) => this._dispatchRunRequests(meshId, frames)),
|
|
1302
1398
|
this._streamWatch(`chat-list:${meshId}`, () => this._engine.chatsWatch({ meshId }), (frames) => this._reclaimDeletedChats(frames)),
|
|
1303
|
-
this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this.
|
|
1399
|
+
this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this._followAgentEdits(meshId, frames))
|
|
1304
1400
|
];
|
|
1305
1401
|
return {
|
|
1306
1402
|
start: () => watches.forEach((watch) => watch.start()),
|
|
@@ -1612,7 +1708,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1612
1708
|
model,
|
|
1613
1709
|
log: this._log
|
|
1614
1710
|
});
|
|
1615
|
-
await this._compactIfNearFull({
|
|
1711
|
+
const folded = await this._compactIfNearFull({
|
|
1616
1712
|
chatId,
|
|
1617
1713
|
transcript,
|
|
1618
1714
|
trigger: 'before-run',
|
|
@@ -1653,8 +1749,12 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1653
1749
|
await this._reconcileAttribution(chatId, senders, holder);
|
|
1654
1750
|
}
|
|
1655
1751
|
const stale = staleToolRunIds(history, priorAsks ?? [], runId);
|
|
1656
|
-
const sentRuns = await this._sentRuns(chatId, holder);
|
|
1657
|
-
const messages = historyFromChunks(history, stale, {
|
|
1752
|
+
const { sentRuns, askedRuns } = await this._sentRuns(chatId, holder);
|
|
1753
|
+
const messages = historyFromChunks(history, stale, {
|
|
1754
|
+
senderNames: senders,
|
|
1755
|
+
sentRuns,
|
|
1756
|
+
askedRuns
|
|
1757
|
+
});
|
|
1658
1758
|
const typedText = currentRepair
|
|
1659
1759
|
? undefined
|
|
1660
1760
|
: history.findLast((chunk) => chunk.userMessage)?.userMessage?.text;
|
|
@@ -1672,12 +1772,11 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1672
1772
|
}
|
|
1673
1773
|
: configuredAgent;
|
|
1674
1774
|
const sendsFiles = lender === null;
|
|
1675
|
-
const
|
|
1676
|
-
|
|
1677
|
-
? this._harness.supportsImagesFor(model)
|
|
1678
|
-
: (this._harness.supportsImages ?? this._info.supportsImages) !== false);
|
|
1775
|
+
const carriesMedia = messages.some((turn) => turn.images?.length || turn.pdfs?.length || turn.videos?.length);
|
|
1776
|
+
const seesImages = sendsFiles && carriesMedia && (await this._seesImages(model));
|
|
1679
1777
|
let unreadableImages = new Map();
|
|
1680
|
-
|
|
1778
|
+
const sampled = [];
|
|
1779
|
+
let pdfPages = { rendered: new Set(), cuts: [] };
|
|
1681
1780
|
if (!seesImages) {
|
|
1682
1781
|
const dropped = messages.reduce((count, turn) => count + (turn.images?.length ?? 0), 0);
|
|
1683
1782
|
if (dropped && (this._harness.supportsImages ?? this._info.supportsImages)) {
|
|
@@ -1781,6 +1880,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1781
1880
|
let runVerdict = 'completed';
|
|
1782
1881
|
let sentRun = null;
|
|
1783
1882
|
let inputLength = 0;
|
|
1883
|
+
let asked;
|
|
1784
1884
|
const writeText = async (type, raw) => {
|
|
1785
1885
|
const text = type === 'content' ? sourceTagFilter.push(raw) : raw;
|
|
1786
1886
|
if (!text) {
|
|
@@ -1868,20 +1968,37 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1868
1968
|
const injected = knowledge
|
|
1869
1969
|
? await this._injectKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords })
|
|
1870
1970
|
: { documentsInjected: false, sourcesWritten: false, ragInjected: false };
|
|
1871
|
-
const hasAudio = messages.some((turn) => turn.audioAttachments?.length);
|
|
1971
|
+
const hasAudio = messages.some((turn) => turn.audioAttachments?.length || turn.videos?.length);
|
|
1872
1972
|
if (hasAudio) {
|
|
1873
1973
|
modelWatch.transcribe(true);
|
|
1874
1974
|
}
|
|
1875
1975
|
await this._hydrateAudio(messages, controller.signal);
|
|
1976
|
+
if (messages.some((turn) => turn.videos?.length)) {
|
|
1977
|
+
await this._hydrateVideos(messages, {
|
|
1978
|
+
signal: controller.signal,
|
|
1979
|
+
runId,
|
|
1980
|
+
seesImages,
|
|
1981
|
+
lent: !sendsFiles,
|
|
1982
|
+
reading: await this._contextReading({ chatId, meshId, folded }, configuredAgent.ctxSize),
|
|
1983
|
+
sampled
|
|
1984
|
+
});
|
|
1985
|
+
}
|
|
1876
1986
|
if (hasAudio) {
|
|
1877
1987
|
modelWatch.transcribe(false);
|
|
1878
1988
|
}
|
|
1879
1989
|
backfillImageOnlyTurns(messages);
|
|
1880
1990
|
if (seesImages) {
|
|
1881
|
-
|
|
1991
|
+
pdfPages = await this._hydratePdfPages(messages, {
|
|
1992
|
+
signal: controller.signal,
|
|
1993
|
+
runId,
|
|
1994
|
+
window: { ctxSize: configuredAgent.ctxSize, modelName: model }
|
|
1995
|
+
});
|
|
1882
1996
|
}
|
|
1883
1997
|
annotateDocumentAttachments(messages);
|
|
1884
|
-
const
|
|
1998
|
+
for (const { turn, notice } of pdfPages.cuts) {
|
|
1999
|
+
turn.content = `${turn.content}\n${notice}`;
|
|
2000
|
+
}
|
|
2001
|
+
const unreadIds = attachTurnIds.filter((id) => !pdfPages.rendered.has(id));
|
|
1885
2002
|
if (!diagramRepair && unreadIds.length > 0 && !injected.documentsInjected) {
|
|
1886
2003
|
const reason = documentUnreadReason({
|
|
1887
2004
|
knowledgeOn: toolsetEnabled('knowledge', runAgent.disabledToolsets),
|
|
@@ -1920,6 +2037,8 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1920
2037
|
cacheKey,
|
|
1921
2038
|
turns: messages.length
|
|
1922
2039
|
});
|
|
2040
|
+
restoreSentContent(messages);
|
|
2041
|
+
asked = messages.findLast((turn) => turn.role === 'user')?.content;
|
|
1923
2042
|
const input = nameSenders(messages);
|
|
1924
2043
|
inputLength = input.length;
|
|
1925
2044
|
const inspecting = await this._capturingInferenceCalls();
|
|
@@ -2070,6 +2189,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2070
2189
|
await this._recordSentRun(chatId, {
|
|
2071
2190
|
runId,
|
|
2072
2191
|
turns: sentRun.turns(inputLength),
|
|
2192
|
+
asked,
|
|
2073
2193
|
holder
|
|
2074
2194
|
});
|
|
2075
2195
|
sentRun = null;
|
|
@@ -2079,6 +2199,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2079
2199
|
metrics: citedTitles ? { ...metrics, citedTitles } : metrics
|
|
2080
2200
|
});
|
|
2081
2201
|
titleOutcome = 'completed';
|
|
2202
|
+
await this._storeVideoDigests(sampled, { meshId, chatId, runId, deviceId });
|
|
2082
2203
|
}
|
|
2083
2204
|
catch (err) {
|
|
2084
2205
|
const outcome = controller.signal.aborted
|
|
@@ -2336,9 +2457,18 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2336
2457
|
return null;
|
|
2337
2458
|
}
|
|
2338
2459
|
}
|
|
2339
|
-
async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId }) {
|
|
2460
|
+
async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId, sourceId }) {
|
|
2340
2461
|
const duplex = this._engine.uploadAttachment();
|
|
2341
|
-
duplex.write({
|
|
2462
|
+
duplex.write({
|
|
2463
|
+
chatId,
|
|
2464
|
+
runId,
|
|
2465
|
+
originDeviceId: deviceId,
|
|
2466
|
+
fileName,
|
|
2467
|
+
mimeType,
|
|
2468
|
+
raw,
|
|
2469
|
+
meshId,
|
|
2470
|
+
sourceId
|
|
2471
|
+
});
|
|
2342
2472
|
const buf = b4a.toBuffer(bytes);
|
|
2343
2473
|
for (let i = 0; i < buf.length; i += UPLOAD_FRAME_BYTES) {
|
|
2344
2474
|
duplex.write({ data: buf.subarray(i, i + UPLOAD_FRAME_BYTES) });
|
|
@@ -2404,6 +2534,259 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2404
2534
|
return unheardAudioNotice(label, 'transcription failed on this device');
|
|
2405
2535
|
}
|
|
2406
2536
|
}
|
|
2537
|
+
async _seesImages(model) {
|
|
2538
|
+
if (this._harness.supportsImagesFor) {
|
|
2539
|
+
return this._harness.supportsImagesFor(model);
|
|
2540
|
+
}
|
|
2541
|
+
const anyModel = (this._harness.supportsImages ?? this._info.supportsImages) !== false;
|
|
2542
|
+
if (!anyModel || !this._harness.list) {
|
|
2543
|
+
return anyModel;
|
|
2544
|
+
}
|
|
2545
|
+
const rows = await this._harness.list({ type: 'llm' }).catch(noop);
|
|
2546
|
+
if (!rows) {
|
|
2547
|
+
return anyModel;
|
|
2548
|
+
}
|
|
2549
|
+
const row = rows.find((candidate) => candidate.name === model);
|
|
2550
|
+
return row ? (row.supportsImages ?? anyModel) : false;
|
|
2551
|
+
}
|
|
2552
|
+
async _hydrateVideos(messages, run) {
|
|
2553
|
+
const { signal, runId, seesImages, lent, reading, sampled } = run;
|
|
2554
|
+
const turns = messages.filter((turn) => turn.videos?.length);
|
|
2555
|
+
if (!turns.length) {
|
|
2556
|
+
return;
|
|
2557
|
+
}
|
|
2558
|
+
const asked = messages.findLast((turn) => turn.role === 'user')?.videos ?? [];
|
|
2559
|
+
const dir = this._imageCacheDir;
|
|
2560
|
+
await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
|
|
2561
|
+
const pinned = this._runImageCacheNames.get(runId) ?? new Set();
|
|
2562
|
+
this._runImageCacheNames.set(runId, pinned);
|
|
2563
|
+
const unsampled = turns.flatMap((turn) => turn.videos ?? []).filter((ref) => !ref.digest);
|
|
2564
|
+
const budget = { ...reading, videos: unsampled.length };
|
|
2565
|
+
for (const turn of turns) {
|
|
2566
|
+
for (const ref of turn.videos ?? []) {
|
|
2567
|
+
if (signal.aborted)
|
|
2568
|
+
return;
|
|
2569
|
+
const hydration = { dir, ref, pinned, seesImages, lent, signal, budget, sampled };
|
|
2570
|
+
const blocks = await this._hydrateVideo(turn, hydration);
|
|
2571
|
+
turn.content = [turn.content.trim(), ...blocks].filter(Boolean).join('\n\n');
|
|
2572
|
+
}
|
|
2573
|
+
delete turn.videos;
|
|
2574
|
+
}
|
|
2575
|
+
const pictured = asked.some((ref) => ref.digest || this._videoProbes.get(ref.blobId)?.hasVideo);
|
|
2576
|
+
if (!seesImages && !lent && pictured) {
|
|
2577
|
+
throw new Error(RUN_FAILURE_REASONS.videoNeedsVision);
|
|
2578
|
+
}
|
|
2579
|
+
}
|
|
2580
|
+
async _hydrateVideo(turn, hydration) {
|
|
2581
|
+
const { ref, seesImages, lent, signal } = hydration;
|
|
2582
|
+
const source = { opened: null };
|
|
2583
|
+
const open = () => (source.opened ??= this._openVideo(hydration));
|
|
2584
|
+
try {
|
|
2585
|
+
const video = await this._readVideo(hydration, open);
|
|
2586
|
+
if (video === 'unsynced') {
|
|
2587
|
+
return [unsyncedVideoNotice(ref.fileName)];
|
|
2588
|
+
}
|
|
2589
|
+
if (video === 'audio') {
|
|
2590
|
+
return [await this._transcribeAttachment(ref, signal)];
|
|
2591
|
+
}
|
|
2592
|
+
if (!seesImages || !('files' in video)) {
|
|
2593
|
+
const notice = lent ? lentVideoNotice(ref.fileName) : hiddenVideoNotice(ref.fileName);
|
|
2594
|
+
return [notice, videoAudioBlock(ref.fileName, video)];
|
|
2595
|
+
}
|
|
2596
|
+
attachPages(turn, video.files);
|
|
2597
|
+
const annotation = videoAnnotation({
|
|
2598
|
+
fileName: ref.fileName,
|
|
2599
|
+
duration: video.duration,
|
|
2600
|
+
times: video.times
|
|
2601
|
+
});
|
|
2602
|
+
return [annotation, videoAudioBlock(ref.fileName, video)];
|
|
2603
|
+
}
|
|
2604
|
+
catch (err) {
|
|
2605
|
+
if (signal.aborted) {
|
|
2606
|
+
throw err;
|
|
2607
|
+
}
|
|
2608
|
+
this._log?.warn('assistant', `video hydration failed — ${ref.fileName}`, err);
|
|
2609
|
+
return [unreadVideoNotice(ref.fileName)];
|
|
2610
|
+
}
|
|
2611
|
+
finally {
|
|
2612
|
+
const video = await source.opened?.catch(noop);
|
|
2613
|
+
if (video) {
|
|
2614
|
+
await fs.promises.unlink(video.file).catch(noop);
|
|
2615
|
+
}
|
|
2616
|
+
}
|
|
2617
|
+
}
|
|
2618
|
+
async _readVideo(hydration, open) {
|
|
2619
|
+
const { ref, seesImages, signal } = hydration;
|
|
2620
|
+
const stored = ref.digest ? await this._storedVideo(ref.digest, hydration) : null;
|
|
2621
|
+
if (stored) {
|
|
2622
|
+
return stored;
|
|
2623
|
+
}
|
|
2624
|
+
const probe = await this._probeVideo(ref, open);
|
|
2625
|
+
if (!probe) {
|
|
2626
|
+
return 'unsynced';
|
|
2627
|
+
}
|
|
2628
|
+
if (!probe.hasVideo) {
|
|
2629
|
+
return 'audio';
|
|
2630
|
+
}
|
|
2631
|
+
if (!seesImages) {
|
|
2632
|
+
return probe.hasAudio ? this._hearVideo(ref, open, signal) : { audio: 'none' };
|
|
2633
|
+
}
|
|
2634
|
+
const video = await open();
|
|
2635
|
+
if (!video) {
|
|
2636
|
+
return 'unsynced';
|
|
2637
|
+
}
|
|
2638
|
+
return this._sampleVideo(video, probe, hydration);
|
|
2639
|
+
}
|
|
2640
|
+
async _probeVideo(ref, open) {
|
|
2641
|
+
const known = this._videoProbes.get(ref.blobId);
|
|
2642
|
+
if (known) {
|
|
2643
|
+
return known;
|
|
2644
|
+
}
|
|
2645
|
+
const video = await open();
|
|
2646
|
+
if (!video) {
|
|
2647
|
+
return null;
|
|
2648
|
+
}
|
|
2649
|
+
const probe = await probeVideo(video.file);
|
|
2650
|
+
this._videoProbes.set(ref.blobId, probe);
|
|
2651
|
+
return probe;
|
|
2652
|
+
}
|
|
2653
|
+
async _storedVideo(digest, { dir, ref, pinned, signal }) {
|
|
2654
|
+
const headerName = videoHeaderCacheName(digest.blobId);
|
|
2655
|
+
pinned.add(headerName);
|
|
2656
|
+
const header = await fs.promises
|
|
2657
|
+
.readFile(path.join(dir, headerName), 'utf8')
|
|
2658
|
+
.then(parseVideoDigestHeader, noop);
|
|
2659
|
+
if (header) {
|
|
2660
|
+
const files = header.times.map((_, i) => {
|
|
2661
|
+
const name = videoFrameCacheName(ref, header.times.length, i);
|
|
2662
|
+
pinned.add(name);
|
|
2663
|
+
return path.join(dir, name);
|
|
2664
|
+
});
|
|
2665
|
+
const cached = await Promise.all(files.map((file) => cacheOrTouch(file, noBytes)));
|
|
2666
|
+
if (cached.every(Boolean)) {
|
|
2667
|
+
return { ...header, files };
|
|
2668
|
+
}
|
|
2669
|
+
}
|
|
2670
|
+
const bytes = await this._readBlob(digest.url, signal);
|
|
2671
|
+
if (!bytes) {
|
|
2672
|
+
return null;
|
|
2673
|
+
}
|
|
2674
|
+
const target = { dir, pinned, digestBlobId: digest.blobId };
|
|
2675
|
+
try {
|
|
2676
|
+
return await this._cacheVideo(ref, unpackVideoDigest(bytes), target);
|
|
2677
|
+
}
|
|
2678
|
+
catch (err) {
|
|
2679
|
+
this._log?.warn('assistant', `video digest unreadable — ${ref.fileName}`, err);
|
|
2680
|
+
return null;
|
|
2681
|
+
}
|
|
2682
|
+
}
|
|
2683
|
+
async _cacheVideo(ref, digest, { dir, pinned, digestBlobId }) {
|
|
2684
|
+
const { frames, ...header } = digest;
|
|
2685
|
+
const files = frames.map((_, i) => {
|
|
2686
|
+
const name = videoFrameCacheName(ref, frames.length, i);
|
|
2687
|
+
pinned.add(name);
|
|
2688
|
+
return path.join(dir, name);
|
|
2689
|
+
});
|
|
2690
|
+
await Promise.all(frames.map((frame, i) => cacheOrTouch(files[i], () => Promise.resolve(frame))));
|
|
2691
|
+
if (digestBlobId) {
|
|
2692
|
+
const headerName = videoHeaderCacheName(digestBlobId);
|
|
2693
|
+
pinned.add(headerName);
|
|
2694
|
+
await cacheOrTouch(path.join(dir, headerName), () => Promise.resolve(b4a.from(JSON.stringify(header))));
|
|
2695
|
+
}
|
|
2696
|
+
return { ...header, files };
|
|
2697
|
+
}
|
|
2698
|
+
async _sampleVideo(video, probe, hydration) {
|
|
2699
|
+
const { ref, budget, signal } = hydration;
|
|
2700
|
+
const started = Date.now();
|
|
2701
|
+
const count = videoFrameCount(probe, budget);
|
|
2702
|
+
const times = frameTimes(probe.duration, count);
|
|
2703
|
+
const frames = await sampleFrames(video.file, { times, maxEdge: VIDEO_FRAME_EDGE, probe });
|
|
2704
|
+
const sampleMs = Date.now() - started;
|
|
2705
|
+
const heard = probe.hasAudio
|
|
2706
|
+
? await this._hearVideo(ref, () => Promise.resolve(video), signal)
|
|
2707
|
+
: { audio: 'none' };
|
|
2708
|
+
const digest = {
|
|
2709
|
+
duration: probe.duration,
|
|
2710
|
+
times: frames.map((frame) => frame.time),
|
|
2711
|
+
...heard,
|
|
2712
|
+
frames: frames.map((frame) => frame.jpeg)
|
|
2713
|
+
};
|
|
2714
|
+
if (!hydration.ref.digest) {
|
|
2715
|
+
hydration.sampled.push({ ref, digest });
|
|
2716
|
+
}
|
|
2717
|
+
this._log?.info('assistant', 'video frames', {
|
|
2718
|
+
fileName: ref.fileName,
|
|
2719
|
+
duration: probe.duration,
|
|
2720
|
+
frames: frames.length,
|
|
2721
|
+
budget: count,
|
|
2722
|
+
ctxSize: budget.ctxSize ?? null,
|
|
2723
|
+
cacheTokens: budget.cacheTokens ?? null,
|
|
2724
|
+
audio: heard.audio,
|
|
2725
|
+
sampleMs
|
|
2726
|
+
});
|
|
2727
|
+
return this._cacheVideo(ref, digest, hydration);
|
|
2728
|
+
}
|
|
2729
|
+
async _storeVideoDigests(sampled, target) {
|
|
2730
|
+
const dir = this._imageCacheDir;
|
|
2731
|
+
const pinned = this._runImageCacheNames.get(target.runId) ?? new Set();
|
|
2732
|
+
for (const { ref, digest } of sampled) {
|
|
2733
|
+
try {
|
|
2734
|
+
const blobId = await this._uploadVideoDigest(ref, digest, target);
|
|
2735
|
+
await this._cacheVideo(ref, digest, { dir, pinned, digestBlobId: blobId });
|
|
2736
|
+
}
|
|
2737
|
+
catch (err) {
|
|
2738
|
+
this._log?.warn('assistant', `video digest not stored — ${ref.fileName}`, err);
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
}
|
|
2742
|
+
async _uploadVideoDigest(ref, digest, target) {
|
|
2743
|
+
const attachment = await this._uploadGenerated({
|
|
2744
|
+
fileName: videoDigestFileName(ref.fileName),
|
|
2745
|
+
mimeType: VIDEO_DIGEST_MIME,
|
|
2746
|
+
bytes: packVideoDigest(digest),
|
|
2747
|
+
raw: true
|
|
2748
|
+
}, { ...target, sourceId: ref.id });
|
|
2749
|
+
return attachment.blobId;
|
|
2750
|
+
}
|
|
2751
|
+
async _openVideo({ dir, ref, pinned, signal }) {
|
|
2752
|
+
const bytes = await this._readBlob(ref.url, signal);
|
|
2753
|
+
if (!bytes) {
|
|
2754
|
+
return null;
|
|
2755
|
+
}
|
|
2756
|
+
const name = `${imageCacheName(ref)}.source`;
|
|
2757
|
+
pinned.add(name);
|
|
2758
|
+
const file = path.join(dir, name);
|
|
2759
|
+
await fs.promises.writeFile(file, bytes, { mode: 0o600 });
|
|
2760
|
+
return { bytes, file };
|
|
2761
|
+
}
|
|
2762
|
+
async _hearVideo(ref, open, signal) {
|
|
2763
|
+
const cached = this._audioTranscripts.get(ref.blobId);
|
|
2764
|
+
if (cached !== undefined) {
|
|
2765
|
+
return cached ? { audio: 'speech', transcript: cached } : { audio: 'silent' };
|
|
2766
|
+
}
|
|
2767
|
+
if (!this._harness.transcribe) {
|
|
2768
|
+
return { audio: 'unheard' };
|
|
2769
|
+
}
|
|
2770
|
+
try {
|
|
2771
|
+
const video = await open();
|
|
2772
|
+
if (!video) {
|
|
2773
|
+
return { audio: 'unheard' };
|
|
2774
|
+
}
|
|
2775
|
+
const model = mediaModelNameFor('transcription', await this._persistedProfile());
|
|
2776
|
+
await this.ensureMediaModelUnpaused('transcription', model);
|
|
2777
|
+
await this._downloads?.waitUntilCached(model, signal);
|
|
2778
|
+
const transcript = (await this._harness.transcribe({ model: '', audio: b4a.toBuffer(video.bytes), signal })).trim();
|
|
2779
|
+
this._audioTranscripts.set(ref.blobId, transcript);
|
|
2780
|
+
return transcript ? { audio: 'speech', transcript } : { audio: 'silent' };
|
|
2781
|
+
}
|
|
2782
|
+
catch (err) {
|
|
2783
|
+
if (signal.aborted) {
|
|
2784
|
+
throw err;
|
|
2785
|
+
}
|
|
2786
|
+
this._log?.warn('assistant', `video transcription failed — ${ref.fileName}`, err);
|
|
2787
|
+
return { audio: 'unheard' };
|
|
2788
|
+
}
|
|
2789
|
+
}
|
|
2407
2790
|
async _hydrateImages(messages, signal, runId) {
|
|
2408
2791
|
const unreadable = new Map();
|
|
2409
2792
|
const turns = messages.filter((turn) => turn.images?.length);
|
|
@@ -2454,36 +2837,66 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2454
2837
|
});
|
|
2455
2838
|
return file;
|
|
2456
2839
|
}
|
|
2457
|
-
async _hydratePdfPages(messages, signal, runId) {
|
|
2840
|
+
async _hydratePdfPages(messages, { signal, runId, window }) {
|
|
2458
2841
|
const rendered = new Set();
|
|
2842
|
+
const cuts = [];
|
|
2459
2843
|
const turns = messages.filter((turn) => turn.pdfs?.length);
|
|
2460
2844
|
if (!turns.length) {
|
|
2461
|
-
return rendered;
|
|
2845
|
+
return { rendered, cuts };
|
|
2462
2846
|
}
|
|
2463
2847
|
const dir = this._imageCacheDir;
|
|
2464
2848
|
await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
|
|
2465
2849
|
const pinned = this._runImageCacheNames.get(runId) ?? new Set();
|
|
2466
2850
|
this._runImageCacheNames.set(runId, pinned);
|
|
2467
2851
|
const latestUser = messages.findLast((turn) => turn.role === 'user');
|
|
2852
|
+
const maxPages = await this._pdfPageBudget(window);
|
|
2468
2853
|
for (const turn of turns) {
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
});
|
|
2479
|
-
budget -= pages.tried;
|
|
2480
|
-
if (turn === latestUser && pages.names.length) {
|
|
2481
|
-
rendered.add(pdf.id);
|
|
2854
|
+
const { shown, notices } = await this._hydrateTurnPdfs(turn, {
|
|
2855
|
+
dir,
|
|
2856
|
+
pinned,
|
|
2857
|
+
signal,
|
|
2858
|
+
maxPages
|
|
2859
|
+
});
|
|
2860
|
+
if (turn === latestUser) {
|
|
2861
|
+
for (const id of shown) {
|
|
2862
|
+
rendered.add(id);
|
|
2482
2863
|
}
|
|
2483
2864
|
}
|
|
2484
|
-
|
|
2865
|
+
cuts.push(...notices.map((notice) => ({ turn, notice })));
|
|
2866
|
+
}
|
|
2867
|
+
return { rendered, cuts };
|
|
2868
|
+
}
|
|
2869
|
+
async _pdfPageBudget({ ctxSize, modelName }) {
|
|
2870
|
+
const custom = await this._customModel(modelName);
|
|
2871
|
+
const window = effectiveCtxSize({ ctxSize, modelName }, { appCtxSize: 0, platform: this._platform, custom });
|
|
2872
|
+
return pdfPageBudget(window);
|
|
2873
|
+
}
|
|
2874
|
+
async _customModel(name) {
|
|
2875
|
+
for await (const list of this._engine.customModelsWatch({})) {
|
|
2876
|
+
return list.models?.find((row) => row.name === name) ?? null;
|
|
2877
|
+
}
|
|
2878
|
+
return null;
|
|
2879
|
+
}
|
|
2880
|
+
async _hydrateTurnPdfs(turn, { dir, pinned, signal, maxPages }) {
|
|
2881
|
+
const shown = [];
|
|
2882
|
+
const notices = [];
|
|
2883
|
+
let budget = maxPages;
|
|
2884
|
+
for (const pdf of turn.pdfs ?? []) {
|
|
2885
|
+
if (budget <= 0) {
|
|
2886
|
+
notices.push(pdfPagesSkippedNotice(pdf.fileName));
|
|
2887
|
+
continue;
|
|
2888
|
+
}
|
|
2889
|
+
const pages = await this._tryHydratePdf(dir, turn, { pdf, pinned, maxPages: budget, signal });
|
|
2890
|
+
budget -= pages.tried;
|
|
2891
|
+
if (pages.names.length) {
|
|
2892
|
+
shown.push(pdf.id);
|
|
2893
|
+
}
|
|
2894
|
+
if (pages.cut) {
|
|
2895
|
+
notices.push(pdfPagesCutNotice({ fileName: pdf.fileName, ...pages.cut }));
|
|
2896
|
+
}
|
|
2485
2897
|
}
|
|
2486
|
-
|
|
2898
|
+
delete turn.pdfs;
|
|
2899
|
+
return { shown, notices };
|
|
2487
2900
|
}
|
|
2488
2901
|
async _tryHydratePdf(dir, turn, opts) {
|
|
2489
2902
|
try {
|
|
@@ -2533,8 +2946,9 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2533
2946
|
try {
|
|
2534
2947
|
const names = [];
|
|
2535
2948
|
let tried = 0;
|
|
2536
|
-
|
|
2537
|
-
|
|
2949
|
+
let page = 0;
|
|
2950
|
+
const pageCount = doc.pageCount();
|
|
2951
|
+
for (; page < pageCount && tried < maxPages; page++) {
|
|
2538
2952
|
if (signal.aborted)
|
|
2539
2953
|
throw new Error('hydration aborted');
|
|
2540
2954
|
const outcome = await this._renderPdfPage(doc, { dir, pdf, page, pinned, signal });
|
|
@@ -2545,7 +2959,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2545
2959
|
names.push(outcome.name);
|
|
2546
2960
|
}
|
|
2547
2961
|
}
|
|
2548
|
-
return { names, tried };
|
|
2962
|
+
return { names, tried, cut: pdfCut(doc, page) };
|
|
2549
2963
|
}
|
|
2550
2964
|
finally {
|
|
2551
2965
|
doc.close();
|
|
@@ -2633,6 +3047,16 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2633
3047
|
}
|
|
2634
3048
|
}
|
|
2635
3049
|
}
|
|
3050
|
+
async _contextReading({ chatId, meshId, folded }, ctxSize) {
|
|
3051
|
+
const last = folded ? {} : await this._lastRunMetrics(chatId, meshId);
|
|
3052
|
+
const window = ctxSize || last.ctxSize;
|
|
3053
|
+
if (!window) {
|
|
3054
|
+
return {};
|
|
3055
|
+
}
|
|
3056
|
+
return last.ctxSize === window
|
|
3057
|
+
? { ctxSize: window, cacheTokens: last.cacheTokens }
|
|
3058
|
+
: { ctxSize: window };
|
|
3059
|
+
}
|
|
2636
3060
|
async _lastRunMetrics(chatId, meshId) {
|
|
2637
3061
|
const deviceId = this._requireDeviceId();
|
|
2638
3062
|
const { chunks } = await meshScoped(this._engine, meshId)
|