@qvac/core 0.10.2 → 0.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -1
- package/dist-lib/lib/api/agent-folders.d.ts +22 -0
- package/dist-lib/lib/api/agent-folders.js +115 -0
- package/dist-lib/lib/api/agents.d.ts +3 -1
- package/dist-lib/lib/api/agents.js +4 -2
- package/dist-lib/lib/api/index.d.ts +1 -0
- package/dist-lib/lib/api/index.js +4 -0
- package/dist-lib/lib/assistant/history.js +18 -2
- package/dist-lib/lib/assistant/message.d.ts +17 -0
- package/dist-lib/lib/assistant/message.js +28 -0
- package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
- package/dist-lib/lib/assistant/video-budget.js +29 -0
- package/dist-lib/lib/assistant-tree.js +2 -1
- package/dist-lib/lib/assistant.d.ts +22 -2
- package/dist-lib/lib/assistant.js +377 -15
- package/dist-lib/lib/core/chat-fork.js +1 -0
- package/dist-lib/lib/core/chat-search.js +1 -1
- package/dist-lib/lib/core/chat.d.ts +1 -0
- package/dist-lib/lib/core/chat.js +2 -1
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +5 -1
- package/dist-lib/lib/core/document-attachment.js +3 -0
- package/dist-lib/lib/core/files/index.js +4 -4
- package/dist-lib/lib/core/files/records/chunks.js +21 -0
- package/dist-lib/lib/core/files/records/purge.js +4 -3
- package/dist-lib/lib/core/files/records/source.js +3 -5
- package/dist-lib/lib/core/files/upload.js +3 -2
- package/dist-lib/lib/core/video/attachment.d.ts +1 -0
- package/dist-lib/lib/core/video/attachment.js +7 -0
- package/dist-lib/lib/core/video/digest.d.ts +16 -0
- package/dist-lib/lib/core/video/digest.js +57 -0
- package/dist-lib/lib/core/video/frames.d.ts +32 -0
- package/dist-lib/lib/core/video/frames.js +224 -0
- package/dist-lib/lib/core.js +25 -5
- package/dist-lib/lib/credential-pusher.d.ts +5 -0
- package/dist-lib/lib/credential-pusher.js +42 -11
- package/dist-lib/lib/errors.d.ts +20 -0
- package/dist-lib/lib/errors.js +20 -0
- package/dist-lib/lib/lent-harness.d.ts +2 -0
- package/dist-lib/lib/lent-harness.js +51 -0
- package/dist-lib/lib/mesh-scoped-engine.js +3 -0
- package/dist-lib/lib/run-notices.d.ts +1 -0
- package/dist-lib/lib/run-notices.js +2 -1
- package/dist-lib/lib/task.js +1 -1
- package/dist-lib/schema/engine.js +53 -2
- package/dist-lib/schema/indexes.js +2 -2
- package/dist-lib/schema/qvac.js +4 -2
- package/dist-lib/spec/bind.js +9 -0
- package/dist-lib/spec/calls.js +3 -0
- package/dist-lib/spec/capabilities.d.ts +6 -0
- package/dist-lib/spec/hrpc/hrpc.json +39 -0
- package/dist-lib/spec/hrpc/index.js +39 -3
- package/dist-lib/spec/hrpc/messages.js +177 -6
- package/dist-lib/spec/hyperschema/index.js +177 -6
- package/dist-lib/spec/hyperschema/schema.json +126 -0
- package/dist-lib/spec/hyperschema/types.d.ts +35 -0
- package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
- package/package.json +4 -1
- package/tui/README.md +10 -0
- package/tui/app.mjs +63 -2
- package/tui/lib/folders-command.mjs +92 -0
- package/tui/lib/knowledge-import.mjs +3 -21
- package/tui/lib/local-path.mjs +21 -0
- package/tui/lib/render.mjs +5 -0
- package/tui/main.mjs +8 -0
|
@@ -44,7 +44,10 @@ import { isLifecycleTeardown, knownAbortOutcome, RUN_ABORT_REASONS, RUN_FAILURE_
|
|
|
44
44
|
import { unref } from './unref.js';
|
|
45
45
|
import { encodeAgentSkillReadiness } from './skill-readiness.js';
|
|
46
46
|
import { harnessClearKvCache, harnessDeathReason, harnessDeleteModel, harnessDied, harnessListModels, harnessPreviewModelSource, harnessSkillCatalog, harnessSkillManagement, harnessDetectLanguage, harnessSpeak, harnessSpeakStream, harnessTools, harnessTranscribe, heldLoadModel, droppedHeldLoadModel, imageDownloadProgress, isHarnessEvent, modelCatalogChanged, toolReportEntry } from './assistant/harness.js';
|
|
47
|
-
import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders } from './assistant/message.js';
|
|
47
|
+
import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders, hiddenVideoNotice, lentVideoNotice, unreadVideoNotice, unsyncedVideoNotice, VIDEO_NO_AUDIO, VIDEO_SILENT, videoAnnotation, videoTranscriptBlock } from './assistant/message.js';
|
|
48
|
+
import { frameTimes, probeVideo, sampleFrames } from './core/video/frames.js';
|
|
49
|
+
import { packVideoDigest, parseVideoDigestHeader, unpackVideoDigest, VIDEO_DIGEST_MIME, videoDigestFileName } from './core/video/digest.js';
|
|
50
|
+
import { VIDEO_FRAME_EDGE, videoFrameCount } from './assistant/video-budget.js';
|
|
48
51
|
import { drainBody, raceInactivity } from './assistant/fetch-body.js';
|
|
49
52
|
import { GENERIC_ACKNOWLEDGEMENTS, HISTORY_TYPES, SKILL_TOOL, TERMINAL_STATES, couldNameSenders, historyFromChunks, inputAvailableToRun, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds } from './assistant/history.js';
|
|
50
53
|
import { errorMessage, isRecord } from './guards.js';
|
|
@@ -59,6 +62,7 @@ const QvacState = local.getEnum('@local/qvac-state');
|
|
|
59
62
|
const FINAL_FLUSH_MS = 10_000;
|
|
60
63
|
const UPLOAD_FRAME_BYTES = 64 * 1024;
|
|
61
64
|
const RUN_IDLE_TIMEOUT_MS = 15 * 60_000;
|
|
65
|
+
const MODEL_RELOAD_DELAY_MS = 2_500;
|
|
62
66
|
function idleWatch(controller, ms) {
|
|
63
67
|
let last = 0;
|
|
64
68
|
let parked = 0;
|
|
@@ -114,6 +118,24 @@ function attachPages(turn, files) {
|
|
|
114
118
|
function noBytes() {
|
|
115
119
|
return Promise.resolve(null);
|
|
116
120
|
}
|
|
121
|
+
function videoFrameCacheName(video, count, frame) {
|
|
122
|
+
return imageCacheName({ blobId: video.blobId, fileName: `frames.${count}.f${frame}.jpg` });
|
|
123
|
+
}
|
|
124
|
+
function videoHeaderCacheName(digestBlobId) {
|
|
125
|
+
return imageCacheName({ blobId: digestBlobId, fileName: 'frames.json' });
|
|
126
|
+
}
|
|
127
|
+
function videoAudioBlock(fileName, { audio, transcript }) {
|
|
128
|
+
if (audio === 'none') {
|
|
129
|
+
return VIDEO_NO_AUDIO;
|
|
130
|
+
}
|
|
131
|
+
if (audio === 'silent') {
|
|
132
|
+
return VIDEO_SILENT;
|
|
133
|
+
}
|
|
134
|
+
if (audio === 'speech' && transcript) {
|
|
135
|
+
return videoTranscriptBlock(transcript);
|
|
136
|
+
}
|
|
137
|
+
return unheardAudioNotice(`The audio of the attached video "${fileName}"`, 'transcription failed');
|
|
138
|
+
}
|
|
117
139
|
function pdfPageCacheName(pdf, page) {
|
|
118
140
|
return imageCacheName({ blobId: pdf.blobId, fileName: `${pdf.fileName}.p${page}.jpg` });
|
|
119
141
|
}
|
|
@@ -146,6 +168,11 @@ function isSetUpHere(status) {
|
|
|
146
168
|
function sameNameList(left, right) {
|
|
147
169
|
return left.length === right.length && left.every((name, index) => name === right[index]);
|
|
148
170
|
}
|
|
171
|
+
function sameLoadSettings(a, b) {
|
|
172
|
+
return ((a.ctxSize ?? 0) === (b.ctxSize ?? 0) &&
|
|
173
|
+
(a.kvCacheK ?? '') === (b.kvCacheK ?? '') &&
|
|
174
|
+
(a.kvCacheV ?? '') === (b.kvCacheV ?? ''));
|
|
175
|
+
}
|
|
149
176
|
function withConfigDefaults(agent, config) {
|
|
150
177
|
if (agent.ctxSize || !config?.ctxSize)
|
|
151
178
|
return agent;
|
|
@@ -161,6 +188,7 @@ const RAG_SOURCE_BOUNDS = {
|
|
|
161
188
|
excerptChars: RAG_SOURCE_EXCERPT_CHARS
|
|
162
189
|
};
|
|
163
190
|
const WEB_RAG_SOURCE_BOUNDS = { ...RAG_SOURCE_BOUNDS, maxUrlChars: MAX_RAG_SOURCE_URL_CHARS };
|
|
191
|
+
const NO_FS_GRANTS = { roots: [], write: [] };
|
|
164
192
|
export class QvacAssistant extends ReadyResource {
|
|
165
193
|
_engine;
|
|
166
194
|
_harness;
|
|
@@ -177,6 +205,8 @@ export class QvacAssistant extends ReadyResource {
|
|
|
177
205
|
_drives;
|
|
178
206
|
_blobReadTimeoutMs;
|
|
179
207
|
_runIdleTimeoutMs;
|
|
208
|
+
_modelReloadDelayMs;
|
|
209
|
+
_reloads = new Map();
|
|
180
210
|
_pdfRasterizer;
|
|
181
211
|
_warmAbort = new AbortController();
|
|
182
212
|
_unloading = new Map();
|
|
@@ -217,6 +247,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
217
247
|
_runReplay;
|
|
218
248
|
_lifecycle;
|
|
219
249
|
_audioTranscripts;
|
|
250
|
+
_videoProbes;
|
|
220
251
|
_pdfPages;
|
|
221
252
|
_chats;
|
|
222
253
|
_runImageCacheNames;
|
|
@@ -225,7 +256,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
225
256
|
_modelCatalog = new ModelCatalogRevisionHub();
|
|
226
257
|
_onHarnessStatus;
|
|
227
258
|
_onModelCatalogEvent;
|
|
228
|
-
constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
|
|
259
|
+
constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, modelReloadDelayMs = MODEL_RELOAD_DELAY_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
|
|
229
260
|
super();
|
|
230
261
|
this._engine = engine;
|
|
231
262
|
this._harness = harness;
|
|
@@ -241,6 +272,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
241
272
|
this._preloadCurrentAgentModel = preloadCurrentAgentModel;
|
|
242
273
|
this._blobReadTimeoutMs = blobReadTimeoutMs;
|
|
243
274
|
this._runIdleTimeoutMs = runIdleTimeoutMs;
|
|
275
|
+
this._modelReloadDelayMs = modelReloadDelayMs;
|
|
244
276
|
this._pdfRasterizer = pdfRasterizerOption ?? createPdfRasterizer(logger);
|
|
245
277
|
this._deviceId = null;
|
|
246
278
|
this._info = { toolNames: [], skills: [], supportsImages: false };
|
|
@@ -267,6 +299,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
267
299
|
this._runReplay = new RunReplayStore(runReplayDir ?? persistentDir('run-replay'));
|
|
268
300
|
this._lifecycle = new AbortController();
|
|
269
301
|
this._audioTranscripts = new Map();
|
|
302
|
+
this._videoProbes = new Map();
|
|
270
303
|
this._pdfPages = new Map();
|
|
271
304
|
this._chats = new Map();
|
|
272
305
|
this._runImageCacheNames = new Map();
|
|
@@ -730,6 +763,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
730
763
|
this._harness.off?.('status', this._onHarnessStatus);
|
|
731
764
|
this._harness.off?.('model', this._onModelCatalogEvent);
|
|
732
765
|
this._warmAbort.abort(RUNNER_CLOSED);
|
|
766
|
+
this._cancelModelReloads();
|
|
733
767
|
this._onlineBoot.destroy(new CoreError('ASSISTANT_CLOSING'));
|
|
734
768
|
for (const run of this._runs.values()) {
|
|
735
769
|
run.controller.abort(RUNNER_CLOSED);
|
|
@@ -768,6 +802,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
768
802
|
this._log?.info('assistant', 'suspending', { runs: this._runs.size });
|
|
769
803
|
this._parked.clear();
|
|
770
804
|
this._stopMeshIntake();
|
|
805
|
+
this._cancelModelReloads();
|
|
771
806
|
await this._downloads?.suspend();
|
|
772
807
|
this._meshList?.destroy();
|
|
773
808
|
this._accountFrames?.destroy();
|
|
@@ -850,6 +885,44 @@ export class QvacAssistant extends ReadyResource {
|
|
|
850
885
|
const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
|
|
851
886
|
return rows?.some((row) => row.name === model && row.isCached) === true;
|
|
852
887
|
}
|
|
888
|
+
_scheduleModelReload(agent) {
|
|
889
|
+
const deviceId = this._deviceId;
|
|
890
|
+
if (!this._harness.loadModel ||
|
|
891
|
+
!deviceId ||
|
|
892
|
+
!agent.modelName ||
|
|
893
|
+
!agent.providerDeviceId ||
|
|
894
|
+
!b4a.equals(agent.providerDeviceId, deviceId)) {
|
|
895
|
+
return;
|
|
896
|
+
}
|
|
897
|
+
const name = agent.modelName;
|
|
898
|
+
clearTimeout(this._reloads.get(name));
|
|
899
|
+
const timer = setTimeout(() => {
|
|
900
|
+
this._reloads.delete(name);
|
|
901
|
+
this._reloadLoadedModel(agent).catch((err) => this._log?.warn('models', 'model reload failed', { model: name, err }));
|
|
902
|
+
}, this._modelReloadDelayMs);
|
|
903
|
+
unref(timer);
|
|
904
|
+
this._reloads.set(name, timer);
|
|
905
|
+
}
|
|
906
|
+
async _reloadLoadedModel(agent) {
|
|
907
|
+
if (this.closing || this.suspended || !this._harness.loadModel)
|
|
908
|
+
return;
|
|
909
|
+
const model = runnableModelName(agent.modelName, await this._runnableHost());
|
|
910
|
+
const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
|
|
911
|
+
const busy = [...this._runs.values()].some((run) => run.executing && !run.borrowed && run.model === model);
|
|
912
|
+
if (busy || !rows?.some((row) => row.name === model && row.isLoaded)) {
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
const config = await this._engine.getConfig({}).catch(noop);
|
|
916
|
+
const knobs = withConfigDefaults(agent, config);
|
|
917
|
+
const { loaded, reason } = await this._harness.loadModel({ model, agent: knobs });
|
|
918
|
+
this._log?.info('models', 'model reload requested', { model, loaded, reason });
|
|
919
|
+
}
|
|
920
|
+
_cancelModelReloads() {
|
|
921
|
+
for (const timer of this._reloads.values()) {
|
|
922
|
+
clearTimeout(timer);
|
|
923
|
+
}
|
|
924
|
+
this._reloads.clear();
|
|
925
|
+
}
|
|
853
926
|
async _agentRunning(model, activeAgentId) {
|
|
854
927
|
const deviceId = this._requireDeviceId();
|
|
855
928
|
const meshId = this._accountMeshId;
|
|
@@ -1072,21 +1145,26 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1072
1145
|
.forget(chatId)
|
|
1073
1146
|
.catch((err) => this._log?.warn('assistant', 'inspector records not dropped', { err }));
|
|
1074
1147
|
}
|
|
1075
|
-
async
|
|
1148
|
+
async _followAgentEdits(meshId, frames) {
|
|
1076
1149
|
let known = null;
|
|
1077
1150
|
for await (const frame of frames) {
|
|
1078
1151
|
if (this.closing || this.suspended)
|
|
1079
1152
|
return;
|
|
1080
1153
|
const agents = frame.agents ?? [];
|
|
1081
1154
|
this._followHosted(meshId, agents);
|
|
1082
|
-
const next = new Map(agents.map((agent) => [agent.id, agentRekeyFingerprint(agent)]));
|
|
1083
1155
|
for (const agent of agents) {
|
|
1084
1156
|
const previous = known?.get(agent.id);
|
|
1085
|
-
if (previous
|
|
1157
|
+
if (!previous) {
|
|
1158
|
+
continue;
|
|
1159
|
+
}
|
|
1160
|
+
if (agentRekeyFingerprint(previous) !== agentRekeyFingerprint(agent)) {
|
|
1086
1161
|
void this._reclaimAgentChatCaches(meshId, agent);
|
|
1087
1162
|
}
|
|
1163
|
+
if (!sameLoadSettings(previous, agent)) {
|
|
1164
|
+
this._scheduleModelReload(agent);
|
|
1165
|
+
}
|
|
1088
1166
|
}
|
|
1089
|
-
known =
|
|
1167
|
+
known = new Map(agents.map((agent) => [agent.id, agent]));
|
|
1090
1168
|
}
|
|
1091
1169
|
}
|
|
1092
1170
|
async _reclaimAgentChatCaches(meshId, agent) {
|
|
@@ -1299,7 +1377,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1299
1377
|
const watches = [
|
|
1300
1378
|
this._streamWatch(`run-requests:${meshId}`, () => this._engine.runRequestsWatch({ meshId }), (frames) => this._dispatchRunRequests(meshId, frames)),
|
|
1301
1379
|
this._streamWatch(`chat-list:${meshId}`, () => this._engine.chatsWatch({ meshId }), (frames) => this._reclaimDeletedChats(frames)),
|
|
1302
|
-
this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this.
|
|
1380
|
+
this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this._followAgentEdits(meshId, frames))
|
|
1303
1381
|
];
|
|
1304
1382
|
return {
|
|
1305
1383
|
start: () => watches.forEach((watch) => watch.start()),
|
|
@@ -1611,7 +1689,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1611
1689
|
model,
|
|
1612
1690
|
log: this._log
|
|
1613
1691
|
});
|
|
1614
|
-
await this._compactIfNearFull({
|
|
1692
|
+
const folded = await this._compactIfNearFull({
|
|
1615
1693
|
chatId,
|
|
1616
1694
|
transcript,
|
|
1617
1695
|
trigger: 'before-run',
|
|
@@ -1619,7 +1697,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1619
1697
|
meshId
|
|
1620
1698
|
});
|
|
1621
1699
|
const toolConfigsPromise = this._loadToolConfigs(engine, agent.id);
|
|
1622
|
-
const fsGrantsPromise = this._loadFsGrants(agent.id);
|
|
1700
|
+
const fsGrantsPromise = lender ? Promise.resolve(NO_FS_GRANTS) : this._loadFsGrants(agent.id);
|
|
1623
1701
|
const all = await listHistoryChunks(engine, chatId);
|
|
1624
1702
|
if (agentSwitched(all, request, deviceId)) {
|
|
1625
1703
|
await this._deleteChatKvCache(chatId, 'agent switched');
|
|
@@ -1670,10 +1748,11 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1670
1748
|
enabledSkills: []
|
|
1671
1749
|
}
|
|
1672
1750
|
: configuredAgent;
|
|
1673
|
-
const
|
|
1674
|
-
|
|
1675
|
-
|
|
1751
|
+
const sendsFiles = lender === null;
|
|
1752
|
+
const carriesMedia = messages.some((turn) => turn.images?.length || turn.pdfs?.length || turn.videos?.length);
|
|
1753
|
+
const seesImages = sendsFiles && carriesMedia && (await this._seesImages(model));
|
|
1676
1754
|
let unreadableImages = new Map();
|
|
1755
|
+
const sampled = [];
|
|
1677
1756
|
let renderedPdfIds = new Set();
|
|
1678
1757
|
if (!seesImages) {
|
|
1679
1758
|
const dropped = messages.reduce((count, turn) => count + (turn.images?.length ?? 0), 0);
|
|
@@ -1865,11 +1944,21 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1865
1944
|
const injected = knowledge
|
|
1866
1945
|
? await this._injectKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords })
|
|
1867
1946
|
: { documentsInjected: false, sourcesWritten: false, ragInjected: false };
|
|
1868
|
-
const hasAudio = messages.some((turn) => turn.audioAttachments?.length);
|
|
1947
|
+
const hasAudio = messages.some((turn) => turn.audioAttachments?.length || turn.videos?.length);
|
|
1869
1948
|
if (hasAudio) {
|
|
1870
1949
|
modelWatch.transcribe(true);
|
|
1871
1950
|
}
|
|
1872
1951
|
await this._hydrateAudio(messages, controller.signal);
|
|
1952
|
+
if (messages.some((turn) => turn.videos?.length)) {
|
|
1953
|
+
await this._hydrateVideos(messages, {
|
|
1954
|
+
signal: controller.signal,
|
|
1955
|
+
runId,
|
|
1956
|
+
seesImages,
|
|
1957
|
+
lent: !sendsFiles,
|
|
1958
|
+
reading: await this._contextReading({ chatId, meshId, folded }, configuredAgent.ctxSize),
|
|
1959
|
+
sampled
|
|
1960
|
+
});
|
|
1961
|
+
}
|
|
1873
1962
|
if (hasAudio) {
|
|
1874
1963
|
modelWatch.transcribe(false);
|
|
1875
1964
|
}
|
|
@@ -2076,6 +2165,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2076
2165
|
metrics: citedTitles ? { ...metrics, citedTitles } : metrics
|
|
2077
2166
|
});
|
|
2078
2167
|
titleOutcome = 'completed';
|
|
2168
|
+
await this._storeVideoDigests(sampled, { meshId, chatId, runId, deviceId });
|
|
2079
2169
|
}
|
|
2080
2170
|
catch (err) {
|
|
2081
2171
|
const outcome = controller.signal.aborted
|
|
@@ -2333,9 +2423,18 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2333
2423
|
return null;
|
|
2334
2424
|
}
|
|
2335
2425
|
}
|
|
2336
|
-
async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId }) {
|
|
2426
|
+
async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId, sourceId }) {
|
|
2337
2427
|
const duplex = this._engine.uploadAttachment();
|
|
2338
|
-
duplex.write({
|
|
2428
|
+
duplex.write({
|
|
2429
|
+
chatId,
|
|
2430
|
+
runId,
|
|
2431
|
+
originDeviceId: deviceId,
|
|
2432
|
+
fileName,
|
|
2433
|
+
mimeType,
|
|
2434
|
+
raw,
|
|
2435
|
+
meshId,
|
|
2436
|
+
sourceId
|
|
2437
|
+
});
|
|
2339
2438
|
const buf = b4a.toBuffer(bytes);
|
|
2340
2439
|
for (let i = 0; i < buf.length; i += UPLOAD_FRAME_BYTES) {
|
|
2341
2440
|
duplex.write({ data: buf.subarray(i, i + UPLOAD_FRAME_BYTES) });
|
|
@@ -2401,6 +2500,259 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2401
2500
|
return unheardAudioNotice(label, 'transcription failed on this device');
|
|
2402
2501
|
}
|
|
2403
2502
|
}
|
|
2503
|
+
async _seesImages(model) {
|
|
2504
|
+
if (this._harness.supportsImagesFor) {
|
|
2505
|
+
return this._harness.supportsImagesFor(model);
|
|
2506
|
+
}
|
|
2507
|
+
const anyModel = (this._harness.supportsImages ?? this._info.supportsImages) !== false;
|
|
2508
|
+
if (!anyModel || !this._harness.list) {
|
|
2509
|
+
return anyModel;
|
|
2510
|
+
}
|
|
2511
|
+
const rows = await this._harness.list({ type: 'llm' }).catch(noop);
|
|
2512
|
+
if (!rows) {
|
|
2513
|
+
return anyModel;
|
|
2514
|
+
}
|
|
2515
|
+
const row = rows.find((candidate) => candidate.name === model);
|
|
2516
|
+
return row ? (row.supportsImages ?? anyModel) : false;
|
|
2517
|
+
}
|
|
2518
|
+
async _hydrateVideos(messages, run) {
|
|
2519
|
+
const { signal, runId, seesImages, lent, reading, sampled } = run;
|
|
2520
|
+
const turns = messages.filter((turn) => turn.videos?.length);
|
|
2521
|
+
if (!turns.length) {
|
|
2522
|
+
return;
|
|
2523
|
+
}
|
|
2524
|
+
const asked = messages.findLast((turn) => turn.role === 'user')?.videos ?? [];
|
|
2525
|
+
const dir = this._imageCacheDir;
|
|
2526
|
+
await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
|
|
2527
|
+
const pinned = this._runImageCacheNames.get(runId) ?? new Set();
|
|
2528
|
+
this._runImageCacheNames.set(runId, pinned);
|
|
2529
|
+
const unsampled = turns.flatMap((turn) => turn.videos ?? []).filter((ref) => !ref.digest);
|
|
2530
|
+
const budget = { ...reading, videos: unsampled.length };
|
|
2531
|
+
for (const turn of turns) {
|
|
2532
|
+
for (const ref of turn.videos ?? []) {
|
|
2533
|
+
if (signal.aborted)
|
|
2534
|
+
return;
|
|
2535
|
+
const hydration = { dir, ref, pinned, seesImages, lent, signal, budget, sampled };
|
|
2536
|
+
const blocks = await this._hydrateVideo(turn, hydration);
|
|
2537
|
+
turn.content = [turn.content.trim(), ...blocks].filter(Boolean).join('\n\n');
|
|
2538
|
+
}
|
|
2539
|
+
delete turn.videos;
|
|
2540
|
+
}
|
|
2541
|
+
const pictured = asked.some((ref) => ref.digest || this._videoProbes.get(ref.blobId)?.hasVideo);
|
|
2542
|
+
if (!seesImages && !lent && pictured) {
|
|
2543
|
+
throw new Error(RUN_FAILURE_REASONS.videoNeedsVision);
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
async _hydrateVideo(turn, hydration) {
|
|
2547
|
+
const { ref, seesImages, lent, signal } = hydration;
|
|
2548
|
+
const source = { opened: null };
|
|
2549
|
+
const open = () => (source.opened ??= this._openVideo(hydration));
|
|
2550
|
+
try {
|
|
2551
|
+
const video = await this._readVideo(hydration, open);
|
|
2552
|
+
if (video === 'unsynced') {
|
|
2553
|
+
return [unsyncedVideoNotice(ref.fileName)];
|
|
2554
|
+
}
|
|
2555
|
+
if (video === 'audio') {
|
|
2556
|
+
return [await this._transcribeAttachment(ref, signal)];
|
|
2557
|
+
}
|
|
2558
|
+
if (!seesImages || !('files' in video)) {
|
|
2559
|
+
const notice = lent ? lentVideoNotice(ref.fileName) : hiddenVideoNotice(ref.fileName);
|
|
2560
|
+
return [notice, videoAudioBlock(ref.fileName, video)];
|
|
2561
|
+
}
|
|
2562
|
+
attachPages(turn, video.files);
|
|
2563
|
+
const annotation = videoAnnotation({
|
|
2564
|
+
fileName: ref.fileName,
|
|
2565
|
+
duration: video.duration,
|
|
2566
|
+
times: video.times
|
|
2567
|
+
});
|
|
2568
|
+
return [annotation, videoAudioBlock(ref.fileName, video)];
|
|
2569
|
+
}
|
|
2570
|
+
catch (err) {
|
|
2571
|
+
if (signal.aborted) {
|
|
2572
|
+
throw err;
|
|
2573
|
+
}
|
|
2574
|
+
this._log?.warn('assistant', `video hydration failed — ${ref.fileName}`, err);
|
|
2575
|
+
return [unreadVideoNotice(ref.fileName)];
|
|
2576
|
+
}
|
|
2577
|
+
finally {
|
|
2578
|
+
const video = await source.opened?.catch(noop);
|
|
2579
|
+
if (video) {
|
|
2580
|
+
await fs.promises.unlink(video.file).catch(noop);
|
|
2581
|
+
}
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
async _readVideo(hydration, open) {
|
|
2585
|
+
const { ref, seesImages, signal } = hydration;
|
|
2586
|
+
const stored = ref.digest ? await this._storedVideo(ref.digest, hydration) : null;
|
|
2587
|
+
if (stored) {
|
|
2588
|
+
return stored;
|
|
2589
|
+
}
|
|
2590
|
+
const probe = await this._probeVideo(ref, open);
|
|
2591
|
+
if (!probe) {
|
|
2592
|
+
return 'unsynced';
|
|
2593
|
+
}
|
|
2594
|
+
if (!probe.hasVideo) {
|
|
2595
|
+
return 'audio';
|
|
2596
|
+
}
|
|
2597
|
+
if (!seesImages) {
|
|
2598
|
+
return probe.hasAudio ? this._hearVideo(ref, open, signal) : { audio: 'none' };
|
|
2599
|
+
}
|
|
2600
|
+
const video = await open();
|
|
2601
|
+
if (!video) {
|
|
2602
|
+
return 'unsynced';
|
|
2603
|
+
}
|
|
2604
|
+
return this._sampleVideo(video, probe, hydration);
|
|
2605
|
+
}
|
|
2606
|
+
async _probeVideo(ref, open) {
|
|
2607
|
+
const known = this._videoProbes.get(ref.blobId);
|
|
2608
|
+
if (known) {
|
|
2609
|
+
return known;
|
|
2610
|
+
}
|
|
2611
|
+
const video = await open();
|
|
2612
|
+
if (!video) {
|
|
2613
|
+
return null;
|
|
2614
|
+
}
|
|
2615
|
+
const probe = await probeVideo(video.file);
|
|
2616
|
+
this._videoProbes.set(ref.blobId, probe);
|
|
2617
|
+
return probe;
|
|
2618
|
+
}
|
|
2619
|
+
async _storedVideo(digest, { dir, ref, pinned, signal }) {
|
|
2620
|
+
const headerName = videoHeaderCacheName(digest.blobId);
|
|
2621
|
+
pinned.add(headerName);
|
|
2622
|
+
const header = await fs.promises
|
|
2623
|
+
.readFile(path.join(dir, headerName), 'utf8')
|
|
2624
|
+
.then(parseVideoDigestHeader, noop);
|
|
2625
|
+
if (header) {
|
|
2626
|
+
const files = header.times.map((_, i) => {
|
|
2627
|
+
const name = videoFrameCacheName(ref, header.times.length, i);
|
|
2628
|
+
pinned.add(name);
|
|
2629
|
+
return path.join(dir, name);
|
|
2630
|
+
});
|
|
2631
|
+
const cached = await Promise.all(files.map((file) => cacheOrTouch(file, noBytes)));
|
|
2632
|
+
if (cached.every(Boolean)) {
|
|
2633
|
+
return { ...header, files };
|
|
2634
|
+
}
|
|
2635
|
+
}
|
|
2636
|
+
const bytes = await this._readBlob(digest.url, signal);
|
|
2637
|
+
if (!bytes) {
|
|
2638
|
+
return null;
|
|
2639
|
+
}
|
|
2640
|
+
const target = { dir, pinned, digestBlobId: digest.blobId };
|
|
2641
|
+
try {
|
|
2642
|
+
return await this._cacheVideo(ref, unpackVideoDigest(bytes), target);
|
|
2643
|
+
}
|
|
2644
|
+
catch (err) {
|
|
2645
|
+
this._log?.warn('assistant', `video digest unreadable — ${ref.fileName}`, err);
|
|
2646
|
+
return null;
|
|
2647
|
+
}
|
|
2648
|
+
}
|
|
2649
|
+
async _cacheVideo(ref, digest, { dir, pinned, digestBlobId }) {
|
|
2650
|
+
const { frames, ...header } = digest;
|
|
2651
|
+
const files = frames.map((_, i) => {
|
|
2652
|
+
const name = videoFrameCacheName(ref, frames.length, i);
|
|
2653
|
+
pinned.add(name);
|
|
2654
|
+
return path.join(dir, name);
|
|
2655
|
+
});
|
|
2656
|
+
await Promise.all(frames.map((frame, i) => cacheOrTouch(files[i], () => Promise.resolve(frame))));
|
|
2657
|
+
if (digestBlobId) {
|
|
2658
|
+
const headerName = videoHeaderCacheName(digestBlobId);
|
|
2659
|
+
pinned.add(headerName);
|
|
2660
|
+
await cacheOrTouch(path.join(dir, headerName), () => Promise.resolve(b4a.from(JSON.stringify(header))));
|
|
2661
|
+
}
|
|
2662
|
+
return { ...header, files };
|
|
2663
|
+
}
|
|
2664
|
+
async _sampleVideo(video, probe, hydration) {
|
|
2665
|
+
const { ref, budget, signal } = hydration;
|
|
2666
|
+
const started = Date.now();
|
|
2667
|
+
const count = videoFrameCount(probe, budget);
|
|
2668
|
+
const times = frameTimes(probe.duration, count);
|
|
2669
|
+
const frames = await sampleFrames(video.file, { times, maxEdge: VIDEO_FRAME_EDGE, probe });
|
|
2670
|
+
const sampleMs = Date.now() - started;
|
|
2671
|
+
const heard = probe.hasAudio
|
|
2672
|
+
? await this._hearVideo(ref, () => Promise.resolve(video), signal)
|
|
2673
|
+
: { audio: 'none' };
|
|
2674
|
+
const digest = {
|
|
2675
|
+
duration: probe.duration,
|
|
2676
|
+
times: frames.map((frame) => frame.time),
|
|
2677
|
+
...heard,
|
|
2678
|
+
frames: frames.map((frame) => frame.jpeg)
|
|
2679
|
+
};
|
|
2680
|
+
if (!hydration.ref.digest) {
|
|
2681
|
+
hydration.sampled.push({ ref, digest });
|
|
2682
|
+
}
|
|
2683
|
+
this._log?.info('assistant', 'video frames', {
|
|
2684
|
+
fileName: ref.fileName,
|
|
2685
|
+
duration: probe.duration,
|
|
2686
|
+
frames: frames.length,
|
|
2687
|
+
budget: count,
|
|
2688
|
+
ctxSize: budget.ctxSize ?? null,
|
|
2689
|
+
cacheTokens: budget.cacheTokens ?? null,
|
|
2690
|
+
audio: heard.audio,
|
|
2691
|
+
sampleMs
|
|
2692
|
+
});
|
|
2693
|
+
return this._cacheVideo(ref, digest, hydration);
|
|
2694
|
+
}
|
|
2695
|
+
async _storeVideoDigests(sampled, target) {
|
|
2696
|
+
const dir = this._imageCacheDir;
|
|
2697
|
+
const pinned = this._runImageCacheNames.get(target.runId) ?? new Set();
|
|
2698
|
+
for (const { ref, digest } of sampled) {
|
|
2699
|
+
try {
|
|
2700
|
+
const blobId = await this._uploadVideoDigest(ref, digest, target);
|
|
2701
|
+
await this._cacheVideo(ref, digest, { dir, pinned, digestBlobId: blobId });
|
|
2702
|
+
}
|
|
2703
|
+
catch (err) {
|
|
2704
|
+
this._log?.warn('assistant', `video digest not stored — ${ref.fileName}`, err);
|
|
2705
|
+
}
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
async _uploadVideoDigest(ref, digest, target) {
|
|
2709
|
+
const attachment = await this._uploadGenerated({
|
|
2710
|
+
fileName: videoDigestFileName(ref.fileName),
|
|
2711
|
+
mimeType: VIDEO_DIGEST_MIME,
|
|
2712
|
+
bytes: packVideoDigest(digest),
|
|
2713
|
+
raw: true
|
|
2714
|
+
}, { ...target, sourceId: ref.id });
|
|
2715
|
+
return attachment.blobId;
|
|
2716
|
+
}
|
|
2717
|
+
async _openVideo({ dir, ref, pinned, signal }) {
|
|
2718
|
+
const bytes = await this._readBlob(ref.url, signal);
|
|
2719
|
+
if (!bytes) {
|
|
2720
|
+
return null;
|
|
2721
|
+
}
|
|
2722
|
+
const name = `${imageCacheName(ref)}.source`;
|
|
2723
|
+
pinned.add(name);
|
|
2724
|
+
const file = path.join(dir, name);
|
|
2725
|
+
await fs.promises.writeFile(file, bytes, { mode: 0o600 });
|
|
2726
|
+
return { bytes, file };
|
|
2727
|
+
}
|
|
2728
|
+
async _hearVideo(ref, open, signal) {
|
|
2729
|
+
const cached = this._audioTranscripts.get(ref.blobId);
|
|
2730
|
+
if (cached !== undefined) {
|
|
2731
|
+
return cached ? { audio: 'speech', transcript: cached } : { audio: 'silent' };
|
|
2732
|
+
}
|
|
2733
|
+
if (!this._harness.transcribe) {
|
|
2734
|
+
return { audio: 'unheard' };
|
|
2735
|
+
}
|
|
2736
|
+
try {
|
|
2737
|
+
const video = await open();
|
|
2738
|
+
if (!video) {
|
|
2739
|
+
return { audio: 'unheard' };
|
|
2740
|
+
}
|
|
2741
|
+
const model = mediaModelNameFor('transcription', await this._persistedProfile());
|
|
2742
|
+
await this.ensureMediaModelUnpaused('transcription', model);
|
|
2743
|
+
await this._downloads?.waitUntilCached(model, signal);
|
|
2744
|
+
const transcript = (await this._harness.transcribe({ model: '', audio: b4a.toBuffer(video.bytes), signal })).trim();
|
|
2745
|
+
this._audioTranscripts.set(ref.blobId, transcript);
|
|
2746
|
+
return transcript ? { audio: 'speech', transcript } : { audio: 'silent' };
|
|
2747
|
+
}
|
|
2748
|
+
catch (err) {
|
|
2749
|
+
if (signal.aborted) {
|
|
2750
|
+
throw err;
|
|
2751
|
+
}
|
|
2752
|
+
this._log?.warn('assistant', `video transcription failed — ${ref.fileName}`, err);
|
|
2753
|
+
return { audio: 'unheard' };
|
|
2754
|
+
}
|
|
2755
|
+
}
|
|
2404
2756
|
async _hydrateImages(messages, signal, runId) {
|
|
2405
2757
|
const unreadable = new Map();
|
|
2406
2758
|
const turns = messages.filter((turn) => turn.images?.length);
|
|
@@ -2630,6 +2982,16 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2630
2982
|
}
|
|
2631
2983
|
}
|
|
2632
2984
|
}
|
|
2985
|
+
async _contextReading({ chatId, meshId, folded }, ctxSize) {
|
|
2986
|
+
const last = folded ? {} : await this._lastRunMetrics(chatId, meshId);
|
|
2987
|
+
const window = ctxSize || last.ctxSize;
|
|
2988
|
+
if (!window) {
|
|
2989
|
+
return {};
|
|
2990
|
+
}
|
|
2991
|
+
return last.ctxSize === window
|
|
2992
|
+
? { ctxSize: window, cacheTokens: last.cacheTokens }
|
|
2993
|
+
: { ctxSize: window };
|
|
2994
|
+
}
|
|
2633
2995
|
async _lastRunMetrics(chatId, meshId) {
|
|
2634
2996
|
const deviceId = this._requireDeviceId();
|
|
2635
2997
|
const { chunks } = await meshScoped(this._engine, meshId)
|
|
@@ -166,6 +166,7 @@ async function forkedChunk(ctx, chunk) {
|
|
|
166
166
|
chatId: ctx.chatId,
|
|
167
167
|
runId: chunk.runId && forkedId(ctx, chunk.runId),
|
|
168
168
|
parentId: chunk.parentId && ctx.kept.has(chunk.parentId) ? forkedId(ctx, chunk.parentId) : null,
|
|
169
|
+
sourceId: chunk.sourceId && forkedId(ctx, chunk.sourceId),
|
|
169
170
|
compaction: chunk.compaction && {
|
|
170
171
|
...chunk.compaction,
|
|
171
172
|
throughId: forkedId(ctx, chunk.compaction.throughId)
|
|
@@ -177,7 +177,7 @@ function matchAttachments(chunks, needle, cap) {
|
|
|
177
177
|
if (matches.length >= cap)
|
|
178
178
|
break;
|
|
179
179
|
const attachment = chunk.attachment;
|
|
180
|
-
if (!attachment || (attachment.deletedAt ?? 0) > 0) {
|
|
180
|
+
if (!attachment || (attachment.deletedAt ?? 0) > 0 || chunk.sourceId) {
|
|
181
181
|
continue;
|
|
182
182
|
}
|
|
183
183
|
const found = locate(attachment.fileName, needle);
|
|
@@ -55,4 +55,5 @@ export interface RemoveChatAttachmentsOptions {
|
|
|
55
55
|
export declare function removeChatAttachments(store: Mesh, chatId: string, { deviceId, at }: RemoveChatAttachmentsOptions): Promise<void>;
|
|
56
56
|
export declare function deleteChat(store: Mesh, deviceId: Buffer | null, id: string): Promise<ChatListing>;
|
|
57
57
|
export declare function softDeleteChatsForAgent(store: Mesh, deviceId: Buffer | null, agentId: string): Promise<void>;
|
|
58
|
+
export declare const AGENT_DELETED = "agent-deleted";
|
|
58
59
|
export declare function cancelRunsForAgent(store: Mesh, deviceId: Buffer, agentId: string): Promise<void>;
|
|
@@ -291,8 +291,9 @@ export async function deleteChat(store, deviceId, id) {
|
|
|
291
291
|
export function softDeleteChatsForAgent(store, deviceId, agentId) {
|
|
292
292
|
return eachChatOfAgent(store, agentId, (chatId) => deleteChat(store, deviceId, chatId));
|
|
293
293
|
}
|
|
294
|
+
export const AGENT_DELETED = 'agent-deleted';
|
|
294
295
|
export function cancelRunsForAgent(store, deviceId, agentId) {
|
|
295
|
-
return eachChatOfAgent(store, agentId, (chatId) => cancelActiveRuns(store, chatId, { deviceId, agentId }));
|
|
296
|
+
return eachChatOfAgent(store, agentId, (chatId) => cancelActiveRuns(store, chatId, { deviceId, agentId, reason: AGENT_DELETED }));
|
|
296
297
|
}
|
|
297
298
|
async function eachChatOfAgent(store, agentId, fn) {
|
|
298
299
|
const snap = store.view.snapshot();
|
|
@@ -3,7 +3,7 @@ import type { QvacChunk, QvacUpdateChunkOperation } from '../../spec/mesh/hypers
|
|
|
3
3
|
import type { Mesh, MeshDispatcher } from './mesh.ts';
|
|
4
4
|
import { type Bounds } from './range.ts';
|
|
5
5
|
type ChunkPayload = Pick<QvacChunk, 'userMessage' | 'content' | 'thinking' | 'toolCall' | 'toolResult' | 'attachment' | 'approvalRequest' | 'runStatus' | 'runRequest' | 'sources' | 'compaction'>;
|
|
6
|
-
type AppendChunkInput = Pick<QvacChunk, 'chatId' | 'runId' | 'type' | 'originDeviceId' | 'parentId' | 'effect' | 'intent' | 'display' | 'rawCall' | 'capability' | 'capabilityArgs' | 'enhance'> & ChunkPayload & {
|
|
6
|
+
type AppendChunkInput = Pick<QvacChunk, 'chatId' | 'runId' | 'type' | 'originDeviceId' | 'parentId' | 'effect' | 'intent' | 'display' | 'rawCall' | 'capability' | 'capabilityArgs' | 'enhance' | 'sourceId'> & ChunkPayload & {
|
|
7
7
|
id?: string;
|
|
8
8
|
};
|
|
9
9
|
type UpdateChunkInput = Pick<QvacChunk, 'id' | 'runStatus' | 'attachment'>;
|
|
@@ -40,7 +40,7 @@ export declare function getChunk(store: Mesh, id: string): Promise<QvacChunk | n
|
|
|
40
40
|
export interface AppendedChunk {
|
|
41
41
|
readonly id: string;
|
|
42
42
|
}
|
|
43
|
-
export declare function appendChunk(store: Mesh, { id, chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, ...payload }: AppendChunkInput, into?: MeshDispatcher): Promise<AppendedChunk>;
|
|
43
|
+
export declare function appendChunk(store: Mesh, { id, chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, sourceId, ...payload }: AppendChunkInput, into?: MeshDispatcher): Promise<AppendedChunk>;
|
|
44
44
|
export declare function updateChunk(store: Mesh, { id, runStatus, attachment }: UpdateChunkInput): Promise<QvacChunk>;
|
|
45
45
|
export declare function updateOp(chunk: QvacChunk, { id, runStatus, attachment }: UpdateChunkInput): QvacUpdateChunkOperation;
|
|
46
46
|
export interface ResolveApprovalInput {
|
|
@@ -118,7 +118,7 @@ function assertPayloadMatchesType(type, payload) {
|
|
|
118
118
|
throw new CoreError('CHUNK_PAYLOAD_INVALID', [`${type} carries ${field} and nothing else`]);
|
|
119
119
|
}
|
|
120
120
|
}
|
|
121
|
-
export async function appendChunk(store, { id = randomId(), chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, ...payload }, into = store) {
|
|
121
|
+
export async function appendChunk(store, { id = randomId(), chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, sourceId, ...payload }, into = store) {
|
|
122
122
|
assertPayloadMatchesType(type, payload);
|
|
123
123
|
if (type === 'sources' && !runId) {
|
|
124
124
|
throw new CoreError('CHUNK_PAYLOAD_INVALID', ['a sources chunk requires runId']);
|
|
@@ -129,6 +129,9 @@ export async function appendChunk(store, { id = randomId(), chatId, runId, type,
|
|
|
129
129
|
if (capability && type !== 'run-request') {
|
|
130
130
|
throw new CoreError('CHUNK_PAYLOAD_INVALID', ['only a run-request carries a capability']);
|
|
131
131
|
}
|
|
132
|
+
if (sourceId && type !== 'attachment') {
|
|
133
|
+
throw new CoreError('CHUNK_PAYLOAD_INVALID', ['only an attachment is derived from another']);
|
|
134
|
+
}
|
|
132
135
|
if (type === 'user-message' && !store.ourMemberId) {
|
|
133
136
|
await store.refreshOurMember();
|
|
134
137
|
}
|
|
@@ -154,6 +157,7 @@ export async function appendChunk(store, { id = randomId(), chatId, runId, type,
|
|
|
154
157
|
capability,
|
|
155
158
|
capabilityArgs,
|
|
156
159
|
enhance,
|
|
160
|
+
sourceId,
|
|
157
161
|
createdAt: Date.now(),
|
|
158
162
|
senderMemberId: type === 'user-message' && store.ourDeviceId && b4a.equals(store.ourDeviceId, originDeviceId)
|
|
159
163
|
? store.ourMemberId || undefined
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { isAudioAttachment } from './audio-attachment.js';
|
|
2
2
|
import { classifyRagDocument, RAG_DOCUMENT_KIND } from './files/rag/index.js';
|
|
3
3
|
import { IMAGE_MIMES } from './image/normalize.js';
|
|
4
|
+
import { isVideoAttachment } from './video/attachment.js';
|
|
4
5
|
export function isDocumentAttachment(ref) {
|
|
5
6
|
if (IMAGE_MIMES.has(ref.mimeType.toLowerCase()))
|
|
6
7
|
return false;
|
|
8
|
+
if (isVideoAttachment(ref.fileName, ref.mimeType))
|
|
9
|
+
return false;
|
|
7
10
|
return !isAudioAttachment(ref.fileName, ref.mimeType);
|
|
8
11
|
}
|
|
9
12
|
export function isPdfAttachment(ref) {
|