@qvac/core 0.10.3 → 0.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -1
- package/dist-lib/lib/assistant/history.js +18 -2
- package/dist-lib/lib/assistant/message.d.ts +17 -0
- package/dist-lib/lib/assistant/message.js +28 -0
- package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
- package/dist-lib/lib/assistant/video-budget.js +29 -0
- package/dist-lib/lib/assistant.d.ts +22 -2
- package/dist-lib/lib/assistant.js +374 -15
- package/dist-lib/lib/core/chat-fork.js +1 -0
- package/dist-lib/lib/core/chat-search.js +1 -1
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +5 -1
- package/dist-lib/lib/core/document-attachment.js +3 -0
- package/dist-lib/lib/core/files/index.js +4 -4
- package/dist-lib/lib/core/files/records/chunks.js +21 -0
- package/dist-lib/lib/core/files/records/purge.js +4 -3
- package/dist-lib/lib/core/files/records/source.js +3 -5
- package/dist-lib/lib/core/files/upload.js +3 -2
- package/dist-lib/lib/core/video/attachment.d.ts +1 -0
- package/dist-lib/lib/core/video/attachment.js +7 -0
- package/dist-lib/lib/core/video/digest.d.ts +16 -0
- package/dist-lib/lib/core/video/digest.js +57 -0
- package/dist-lib/lib/core/video/frames.d.ts +32 -0
- package/dist-lib/lib/core/video/frames.js +224 -0
- package/dist-lib/lib/core.js +19 -1
- package/dist-lib/lib/credential-pusher.d.ts +5 -0
- package/dist-lib/lib/credential-pusher.js +42 -11
- package/dist-lib/lib/run-notices.d.ts +1 -0
- package/dist-lib/lib/run-notices.js +2 -1
- package/dist-lib/lib/task.js +1 -1
- package/dist-lib/schema/engine.js +4 -2
- package/dist-lib/schema/indexes.js +2 -2
- package/dist-lib/schema/qvac.js +4 -2
- package/dist-lib/spec/hrpc/messages.js +16 -6
- package/dist-lib/spec/hyperschema/index.js +16 -6
- package/dist-lib/spec/hyperschema/schema.json +10 -0
- package/dist-lib/spec/hyperschema/types.d.ts +2 -0
- package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
- package/package.json +3 -1
- package/tui/app.mjs +34 -1
- package/tui/lib/render.mjs +5 -0
- package/tui/main.mjs +8 -0
|
@@ -44,7 +44,10 @@ import { isLifecycleTeardown, knownAbortOutcome, RUN_ABORT_REASONS, RUN_FAILURE_
|
|
|
44
44
|
import { unref } from './unref.js';
|
|
45
45
|
import { encodeAgentSkillReadiness } from './skill-readiness.js';
|
|
46
46
|
import { harnessClearKvCache, harnessDeathReason, harnessDeleteModel, harnessDied, harnessListModels, harnessPreviewModelSource, harnessSkillCatalog, harnessSkillManagement, harnessDetectLanguage, harnessSpeak, harnessSpeakStream, harnessTools, harnessTranscribe, heldLoadModel, droppedHeldLoadModel, imageDownloadProgress, isHarnessEvent, modelCatalogChanged, toolReportEntry } from './assistant/harness.js';
|
|
47
|
-
import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders } from './assistant/message.js';
|
|
47
|
+
import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders, hiddenVideoNotice, lentVideoNotice, unreadVideoNotice, unsyncedVideoNotice, VIDEO_NO_AUDIO, VIDEO_SILENT, videoAnnotation, videoTranscriptBlock } from './assistant/message.js';
|
|
48
|
+
import { frameTimes, probeVideo, sampleFrames } from './core/video/frames.js';
|
|
49
|
+
import { packVideoDigest, parseVideoDigestHeader, unpackVideoDigest, VIDEO_DIGEST_MIME, videoDigestFileName } from './core/video/digest.js';
|
|
50
|
+
import { VIDEO_FRAME_EDGE, videoFrameCount } from './assistant/video-budget.js';
|
|
48
51
|
import { drainBody, raceInactivity } from './assistant/fetch-body.js';
|
|
49
52
|
import { GENERIC_ACKNOWLEDGEMENTS, HISTORY_TYPES, SKILL_TOOL, TERMINAL_STATES, couldNameSenders, historyFromChunks, inputAvailableToRun, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds } from './assistant/history.js';
|
|
50
53
|
import { errorMessage, isRecord } from './guards.js';
|
|
@@ -59,6 +62,7 @@ const QvacState = local.getEnum('@local/qvac-state');
|
|
|
59
62
|
const FINAL_FLUSH_MS = 10_000;
|
|
60
63
|
const UPLOAD_FRAME_BYTES = 64 * 1024;
|
|
61
64
|
const RUN_IDLE_TIMEOUT_MS = 15 * 60_000;
|
|
65
|
+
const MODEL_RELOAD_DELAY_MS = 2_500;
|
|
62
66
|
function idleWatch(controller, ms) {
|
|
63
67
|
let last = 0;
|
|
64
68
|
let parked = 0;
|
|
@@ -114,6 +118,24 @@ function attachPages(turn, files) {
|
|
|
114
118
|
function noBytes() {
|
|
115
119
|
return Promise.resolve(null);
|
|
116
120
|
}
|
|
121
|
+
function videoFrameCacheName(video, count, frame) {
|
|
122
|
+
return imageCacheName({ blobId: video.blobId, fileName: `frames.${count}.f${frame}.jpg` });
|
|
123
|
+
}
|
|
124
|
+
function videoHeaderCacheName(digestBlobId) {
|
|
125
|
+
return imageCacheName({ blobId: digestBlobId, fileName: 'frames.json' });
|
|
126
|
+
}
|
|
127
|
+
function videoAudioBlock(fileName, { audio, transcript }) {
|
|
128
|
+
if (audio === 'none') {
|
|
129
|
+
return VIDEO_NO_AUDIO;
|
|
130
|
+
}
|
|
131
|
+
if (audio === 'silent') {
|
|
132
|
+
return VIDEO_SILENT;
|
|
133
|
+
}
|
|
134
|
+
if (audio === 'speech' && transcript) {
|
|
135
|
+
return videoTranscriptBlock(transcript);
|
|
136
|
+
}
|
|
137
|
+
return unheardAudioNotice(`The audio of the attached video "${fileName}"`, 'transcription failed');
|
|
138
|
+
}
|
|
117
139
|
function pdfPageCacheName(pdf, page) {
|
|
118
140
|
return imageCacheName({ blobId: pdf.blobId, fileName: `${pdf.fileName}.p${page}.jpg` });
|
|
119
141
|
}
|
|
@@ -146,6 +168,11 @@ function isSetUpHere(status) {
|
|
|
146
168
|
function sameNameList(left, right) {
|
|
147
169
|
return left.length === right.length && left.every((name, index) => name === right[index]);
|
|
148
170
|
}
|
|
171
|
+
function sameLoadSettings(a, b) {
|
|
172
|
+
return ((a.ctxSize ?? 0) === (b.ctxSize ?? 0) &&
|
|
173
|
+
(a.kvCacheK ?? '') === (b.kvCacheK ?? '') &&
|
|
174
|
+
(a.kvCacheV ?? '') === (b.kvCacheV ?? ''));
|
|
175
|
+
}
|
|
149
176
|
function withConfigDefaults(agent, config) {
|
|
150
177
|
if (agent.ctxSize || !config?.ctxSize)
|
|
151
178
|
return agent;
|
|
@@ -178,6 +205,8 @@ export class QvacAssistant extends ReadyResource {
|
|
|
178
205
|
_drives;
|
|
179
206
|
_blobReadTimeoutMs;
|
|
180
207
|
_runIdleTimeoutMs;
|
|
208
|
+
_modelReloadDelayMs;
|
|
209
|
+
_reloads = new Map();
|
|
181
210
|
_pdfRasterizer;
|
|
182
211
|
_warmAbort = new AbortController();
|
|
183
212
|
_unloading = new Map();
|
|
@@ -218,6 +247,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
218
247
|
_runReplay;
|
|
219
248
|
_lifecycle;
|
|
220
249
|
_audioTranscripts;
|
|
250
|
+
_videoProbes;
|
|
221
251
|
_pdfPages;
|
|
222
252
|
_chats;
|
|
223
253
|
_runImageCacheNames;
|
|
@@ -226,7 +256,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
226
256
|
_modelCatalog = new ModelCatalogRevisionHub();
|
|
227
257
|
_onHarnessStatus;
|
|
228
258
|
_onModelCatalogEvent;
|
|
229
|
-
constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
|
|
259
|
+
constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, modelReloadDelayMs = MODEL_RELOAD_DELAY_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
|
|
230
260
|
super();
|
|
231
261
|
this._engine = engine;
|
|
232
262
|
this._harness = harness;
|
|
@@ -242,6 +272,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
242
272
|
this._preloadCurrentAgentModel = preloadCurrentAgentModel;
|
|
243
273
|
this._blobReadTimeoutMs = blobReadTimeoutMs;
|
|
244
274
|
this._runIdleTimeoutMs = runIdleTimeoutMs;
|
|
275
|
+
this._modelReloadDelayMs = modelReloadDelayMs;
|
|
245
276
|
this._pdfRasterizer = pdfRasterizerOption ?? createPdfRasterizer(logger);
|
|
246
277
|
this._deviceId = null;
|
|
247
278
|
this._info = { toolNames: [], skills: [], supportsImages: false };
|
|
@@ -268,6 +299,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
268
299
|
this._runReplay = new RunReplayStore(runReplayDir ?? persistentDir('run-replay'));
|
|
269
300
|
this._lifecycle = new AbortController();
|
|
270
301
|
this._audioTranscripts = new Map();
|
|
302
|
+
this._videoProbes = new Map();
|
|
271
303
|
this._pdfPages = new Map();
|
|
272
304
|
this._chats = new Map();
|
|
273
305
|
this._runImageCacheNames = new Map();
|
|
@@ -731,6 +763,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
731
763
|
this._harness.off?.('status', this._onHarnessStatus);
|
|
732
764
|
this._harness.off?.('model', this._onModelCatalogEvent);
|
|
733
765
|
this._warmAbort.abort(RUNNER_CLOSED);
|
|
766
|
+
this._cancelModelReloads();
|
|
734
767
|
this._onlineBoot.destroy(new CoreError('ASSISTANT_CLOSING'));
|
|
735
768
|
for (const run of this._runs.values()) {
|
|
736
769
|
run.controller.abort(RUNNER_CLOSED);
|
|
@@ -769,6 +802,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
769
802
|
this._log?.info('assistant', 'suspending', { runs: this._runs.size });
|
|
770
803
|
this._parked.clear();
|
|
771
804
|
this._stopMeshIntake();
|
|
805
|
+
this._cancelModelReloads();
|
|
772
806
|
await this._downloads?.suspend();
|
|
773
807
|
this._meshList?.destroy();
|
|
774
808
|
this._accountFrames?.destroy();
|
|
@@ -851,6 +885,44 @@ export class QvacAssistant extends ReadyResource {
|
|
|
851
885
|
const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
|
|
852
886
|
return rows?.some((row) => row.name === model && row.isCached) === true;
|
|
853
887
|
}
|
|
888
|
+
_scheduleModelReload(agent) {
|
|
889
|
+
const deviceId = this._deviceId;
|
|
890
|
+
if (!this._harness.loadModel ||
|
|
891
|
+
!deviceId ||
|
|
892
|
+
!agent.modelName ||
|
|
893
|
+
!agent.providerDeviceId ||
|
|
894
|
+
!b4a.equals(agent.providerDeviceId, deviceId)) {
|
|
895
|
+
return;
|
|
896
|
+
}
|
|
897
|
+
const name = agent.modelName;
|
|
898
|
+
clearTimeout(this._reloads.get(name));
|
|
899
|
+
const timer = setTimeout(() => {
|
|
900
|
+
this._reloads.delete(name);
|
|
901
|
+
this._reloadLoadedModel(agent).catch((err) => this._log?.warn('models', 'model reload failed', { model: name, err }));
|
|
902
|
+
}, this._modelReloadDelayMs);
|
|
903
|
+
unref(timer);
|
|
904
|
+
this._reloads.set(name, timer);
|
|
905
|
+
}
|
|
906
|
+
async _reloadLoadedModel(agent) {
|
|
907
|
+
if (this.closing || this.suspended || !this._harness.loadModel)
|
|
908
|
+
return;
|
|
909
|
+
const model = runnableModelName(agent.modelName, await this._runnableHost());
|
|
910
|
+
const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
|
|
911
|
+
const busy = [...this._runs.values()].some((run) => run.executing && !run.borrowed && run.model === model);
|
|
912
|
+
if (busy || !rows?.some((row) => row.name === model && row.isLoaded)) {
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
const config = await this._engine.getConfig({}).catch(noop);
|
|
916
|
+
const knobs = withConfigDefaults(agent, config);
|
|
917
|
+
const { loaded, reason } = await this._harness.loadModel({ model, agent: knobs });
|
|
918
|
+
this._log?.info('models', 'model reload requested', { model, loaded, reason });
|
|
919
|
+
}
|
|
920
|
+
_cancelModelReloads() {
|
|
921
|
+
for (const timer of this._reloads.values()) {
|
|
922
|
+
clearTimeout(timer);
|
|
923
|
+
}
|
|
924
|
+
this._reloads.clear();
|
|
925
|
+
}
|
|
854
926
|
async _agentRunning(model, activeAgentId) {
|
|
855
927
|
const deviceId = this._requireDeviceId();
|
|
856
928
|
const meshId = this._accountMeshId;
|
|
@@ -1073,21 +1145,26 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1073
1145
|
.forget(chatId)
|
|
1074
1146
|
.catch((err) => this._log?.warn('assistant', 'inspector records not dropped', { err }));
|
|
1075
1147
|
}
|
|
1076
|
-
async
|
|
1148
|
+
async _followAgentEdits(meshId, frames) {
|
|
1077
1149
|
let known = null;
|
|
1078
1150
|
for await (const frame of frames) {
|
|
1079
1151
|
if (this.closing || this.suspended)
|
|
1080
1152
|
return;
|
|
1081
1153
|
const agents = frame.agents ?? [];
|
|
1082
1154
|
this._followHosted(meshId, agents);
|
|
1083
|
-
const next = new Map(agents.map((agent) => [agent.id, agentRekeyFingerprint(agent)]));
|
|
1084
1155
|
for (const agent of agents) {
|
|
1085
1156
|
const previous = known?.get(agent.id);
|
|
1086
|
-
if (previous
|
|
1157
|
+
if (!previous) {
|
|
1158
|
+
continue;
|
|
1159
|
+
}
|
|
1160
|
+
if (agentRekeyFingerprint(previous) !== agentRekeyFingerprint(agent)) {
|
|
1087
1161
|
void this._reclaimAgentChatCaches(meshId, agent);
|
|
1088
1162
|
}
|
|
1163
|
+
if (!sameLoadSettings(previous, agent)) {
|
|
1164
|
+
this._scheduleModelReload(agent);
|
|
1165
|
+
}
|
|
1089
1166
|
}
|
|
1090
|
-
known =
|
|
1167
|
+
known = new Map(agents.map((agent) => [agent.id, agent]));
|
|
1091
1168
|
}
|
|
1092
1169
|
}
|
|
1093
1170
|
async _reclaimAgentChatCaches(meshId, agent) {
|
|
@@ -1300,7 +1377,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1300
1377
|
const watches = [
|
|
1301
1378
|
this._streamWatch(`run-requests:${meshId}`, () => this._engine.runRequestsWatch({ meshId }), (frames) => this._dispatchRunRequests(meshId, frames)),
|
|
1302
1379
|
this._streamWatch(`chat-list:${meshId}`, () => this._engine.chatsWatch({ meshId }), (frames) => this._reclaimDeletedChats(frames)),
|
|
1303
|
-
this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this.
|
|
1380
|
+
this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this._followAgentEdits(meshId, frames))
|
|
1304
1381
|
];
|
|
1305
1382
|
return {
|
|
1306
1383
|
start: () => watches.forEach((watch) => watch.start()),
|
|
@@ -1612,7 +1689,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1612
1689
|
model,
|
|
1613
1690
|
log: this._log
|
|
1614
1691
|
});
|
|
1615
|
-
await this._compactIfNearFull({
|
|
1692
|
+
const folded = await this._compactIfNearFull({
|
|
1616
1693
|
chatId,
|
|
1617
1694
|
transcript,
|
|
1618
1695
|
trigger: 'before-run',
|
|
@@ -1672,11 +1749,10 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1672
1749
|
}
|
|
1673
1750
|
: configuredAgent;
|
|
1674
1751
|
const sendsFiles = lender === null;
|
|
1675
|
-
const
|
|
1676
|
-
|
|
1677
|
-
? this._harness.supportsImagesFor(model)
|
|
1678
|
-
: (this._harness.supportsImages ?? this._info.supportsImages) !== false);
|
|
1752
|
+
const carriesMedia = messages.some((turn) => turn.images?.length || turn.pdfs?.length || turn.videos?.length);
|
|
1753
|
+
const seesImages = sendsFiles && carriesMedia && (await this._seesImages(model));
|
|
1679
1754
|
let unreadableImages = new Map();
|
|
1755
|
+
const sampled = [];
|
|
1680
1756
|
let renderedPdfIds = new Set();
|
|
1681
1757
|
if (!seesImages) {
|
|
1682
1758
|
const dropped = messages.reduce((count, turn) => count + (turn.images?.length ?? 0), 0);
|
|
@@ -1868,11 +1944,21 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1868
1944
|
const injected = knowledge
|
|
1869
1945
|
? await this._injectKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords })
|
|
1870
1946
|
: { documentsInjected: false, sourcesWritten: false, ragInjected: false };
|
|
1871
|
-
const hasAudio = messages.some((turn) => turn.audioAttachments?.length);
|
|
1947
|
+
const hasAudio = messages.some((turn) => turn.audioAttachments?.length || turn.videos?.length);
|
|
1872
1948
|
if (hasAudio) {
|
|
1873
1949
|
modelWatch.transcribe(true);
|
|
1874
1950
|
}
|
|
1875
1951
|
await this._hydrateAudio(messages, controller.signal);
|
|
1952
|
+
if (messages.some((turn) => turn.videos?.length)) {
|
|
1953
|
+
await this._hydrateVideos(messages, {
|
|
1954
|
+
signal: controller.signal,
|
|
1955
|
+
runId,
|
|
1956
|
+
seesImages,
|
|
1957
|
+
lent: !sendsFiles,
|
|
1958
|
+
reading: await this._contextReading({ chatId, meshId, folded }, configuredAgent.ctxSize),
|
|
1959
|
+
sampled
|
|
1960
|
+
});
|
|
1961
|
+
}
|
|
1876
1962
|
if (hasAudio) {
|
|
1877
1963
|
modelWatch.transcribe(false);
|
|
1878
1964
|
}
|
|
@@ -2079,6 +2165,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2079
2165
|
metrics: citedTitles ? { ...metrics, citedTitles } : metrics
|
|
2080
2166
|
});
|
|
2081
2167
|
titleOutcome = 'completed';
|
|
2168
|
+
await this._storeVideoDigests(sampled, { meshId, chatId, runId, deviceId });
|
|
2082
2169
|
}
|
|
2083
2170
|
catch (err) {
|
|
2084
2171
|
const outcome = controller.signal.aborted
|
|
@@ -2336,9 +2423,18 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2336
2423
|
return null;
|
|
2337
2424
|
}
|
|
2338
2425
|
}
|
|
2339
|
-
async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId }) {
|
|
2426
|
+
async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId, sourceId }) {
|
|
2340
2427
|
const duplex = this._engine.uploadAttachment();
|
|
2341
|
-
duplex.write({
|
|
2428
|
+
duplex.write({
|
|
2429
|
+
chatId,
|
|
2430
|
+
runId,
|
|
2431
|
+
originDeviceId: deviceId,
|
|
2432
|
+
fileName,
|
|
2433
|
+
mimeType,
|
|
2434
|
+
raw,
|
|
2435
|
+
meshId,
|
|
2436
|
+
sourceId
|
|
2437
|
+
});
|
|
2342
2438
|
const buf = b4a.toBuffer(bytes);
|
|
2343
2439
|
for (let i = 0; i < buf.length; i += UPLOAD_FRAME_BYTES) {
|
|
2344
2440
|
duplex.write({ data: buf.subarray(i, i + UPLOAD_FRAME_BYTES) });
|
|
@@ -2404,6 +2500,259 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2404
2500
|
return unheardAudioNotice(label, 'transcription failed on this device');
|
|
2405
2501
|
}
|
|
2406
2502
|
}
|
|
2503
|
+
async _seesImages(model) {
|
|
2504
|
+
if (this._harness.supportsImagesFor) {
|
|
2505
|
+
return this._harness.supportsImagesFor(model);
|
|
2506
|
+
}
|
|
2507
|
+
const anyModel = (this._harness.supportsImages ?? this._info.supportsImages) !== false;
|
|
2508
|
+
if (!anyModel || !this._harness.list) {
|
|
2509
|
+
return anyModel;
|
|
2510
|
+
}
|
|
2511
|
+
const rows = await this._harness.list({ type: 'llm' }).catch(noop);
|
|
2512
|
+
if (!rows) {
|
|
2513
|
+
return anyModel;
|
|
2514
|
+
}
|
|
2515
|
+
const row = rows.find((candidate) => candidate.name === model);
|
|
2516
|
+
return row ? (row.supportsImages ?? anyModel) : false;
|
|
2517
|
+
}
|
|
2518
|
+
async _hydrateVideos(messages, run) {
|
|
2519
|
+
const { signal, runId, seesImages, lent, reading, sampled } = run;
|
|
2520
|
+
const turns = messages.filter((turn) => turn.videos?.length);
|
|
2521
|
+
if (!turns.length) {
|
|
2522
|
+
return;
|
|
2523
|
+
}
|
|
2524
|
+
const asked = messages.findLast((turn) => turn.role === 'user')?.videos ?? [];
|
|
2525
|
+
const dir = this._imageCacheDir;
|
|
2526
|
+
await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
|
|
2527
|
+
const pinned = this._runImageCacheNames.get(runId) ?? new Set();
|
|
2528
|
+
this._runImageCacheNames.set(runId, pinned);
|
|
2529
|
+
const unsampled = turns.flatMap((turn) => turn.videos ?? []).filter((ref) => !ref.digest);
|
|
2530
|
+
const budget = { ...reading, videos: unsampled.length };
|
|
2531
|
+
for (const turn of turns) {
|
|
2532
|
+
for (const ref of turn.videos ?? []) {
|
|
2533
|
+
if (signal.aborted)
|
|
2534
|
+
return;
|
|
2535
|
+
const hydration = { dir, ref, pinned, seesImages, lent, signal, budget, sampled };
|
|
2536
|
+
const blocks = await this._hydrateVideo(turn, hydration);
|
|
2537
|
+
turn.content = [turn.content.trim(), ...blocks].filter(Boolean).join('\n\n');
|
|
2538
|
+
}
|
|
2539
|
+
delete turn.videos;
|
|
2540
|
+
}
|
|
2541
|
+
const pictured = asked.some((ref) => ref.digest || this._videoProbes.get(ref.blobId)?.hasVideo);
|
|
2542
|
+
if (!seesImages && !lent && pictured) {
|
|
2543
|
+
throw new Error(RUN_FAILURE_REASONS.videoNeedsVision);
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
async _hydrateVideo(turn, hydration) {
|
|
2547
|
+
const { ref, seesImages, lent, signal } = hydration;
|
|
2548
|
+
const source = { opened: null };
|
|
2549
|
+
const open = () => (source.opened ??= this._openVideo(hydration));
|
|
2550
|
+
try {
|
|
2551
|
+
const video = await this._readVideo(hydration, open);
|
|
2552
|
+
if (video === 'unsynced') {
|
|
2553
|
+
return [unsyncedVideoNotice(ref.fileName)];
|
|
2554
|
+
}
|
|
2555
|
+
if (video === 'audio') {
|
|
2556
|
+
return [await this._transcribeAttachment(ref, signal)];
|
|
2557
|
+
}
|
|
2558
|
+
if (!seesImages || !('files' in video)) {
|
|
2559
|
+
const notice = lent ? lentVideoNotice(ref.fileName) : hiddenVideoNotice(ref.fileName);
|
|
2560
|
+
return [notice, videoAudioBlock(ref.fileName, video)];
|
|
2561
|
+
}
|
|
2562
|
+
attachPages(turn, video.files);
|
|
2563
|
+
const annotation = videoAnnotation({
|
|
2564
|
+
fileName: ref.fileName,
|
|
2565
|
+
duration: video.duration,
|
|
2566
|
+
times: video.times
|
|
2567
|
+
});
|
|
2568
|
+
return [annotation, videoAudioBlock(ref.fileName, video)];
|
|
2569
|
+
}
|
|
2570
|
+
catch (err) {
|
|
2571
|
+
if (signal.aborted) {
|
|
2572
|
+
throw err;
|
|
2573
|
+
}
|
|
2574
|
+
this._log?.warn('assistant', `video hydration failed — ${ref.fileName}`, err);
|
|
2575
|
+
return [unreadVideoNotice(ref.fileName)];
|
|
2576
|
+
}
|
|
2577
|
+
finally {
|
|
2578
|
+
const video = await source.opened?.catch(noop);
|
|
2579
|
+
if (video) {
|
|
2580
|
+
await fs.promises.unlink(video.file).catch(noop);
|
|
2581
|
+
}
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
async _readVideo(hydration, open) {
|
|
2585
|
+
const { ref, seesImages, signal } = hydration;
|
|
2586
|
+
const stored = ref.digest ? await this._storedVideo(ref.digest, hydration) : null;
|
|
2587
|
+
if (stored) {
|
|
2588
|
+
return stored;
|
|
2589
|
+
}
|
|
2590
|
+
const probe = await this._probeVideo(ref, open);
|
|
2591
|
+
if (!probe) {
|
|
2592
|
+
return 'unsynced';
|
|
2593
|
+
}
|
|
2594
|
+
if (!probe.hasVideo) {
|
|
2595
|
+
return 'audio';
|
|
2596
|
+
}
|
|
2597
|
+
if (!seesImages) {
|
|
2598
|
+
return probe.hasAudio ? this._hearVideo(ref, open, signal) : { audio: 'none' };
|
|
2599
|
+
}
|
|
2600
|
+
const video = await open();
|
|
2601
|
+
if (!video) {
|
|
2602
|
+
return 'unsynced';
|
|
2603
|
+
}
|
|
2604
|
+
return this._sampleVideo(video, probe, hydration);
|
|
2605
|
+
}
|
|
2606
|
+
async _probeVideo(ref, open) {
|
|
2607
|
+
const known = this._videoProbes.get(ref.blobId);
|
|
2608
|
+
if (known) {
|
|
2609
|
+
return known;
|
|
2610
|
+
}
|
|
2611
|
+
const video = await open();
|
|
2612
|
+
if (!video) {
|
|
2613
|
+
return null;
|
|
2614
|
+
}
|
|
2615
|
+
const probe = await probeVideo(video.file);
|
|
2616
|
+
this._videoProbes.set(ref.blobId, probe);
|
|
2617
|
+
return probe;
|
|
2618
|
+
}
|
|
2619
|
+
async _storedVideo(digest, { dir, ref, pinned, signal }) {
|
|
2620
|
+
const headerName = videoHeaderCacheName(digest.blobId);
|
|
2621
|
+
pinned.add(headerName);
|
|
2622
|
+
const header = await fs.promises
|
|
2623
|
+
.readFile(path.join(dir, headerName), 'utf8')
|
|
2624
|
+
.then(parseVideoDigestHeader, noop);
|
|
2625
|
+
if (header) {
|
|
2626
|
+
const files = header.times.map((_, i) => {
|
|
2627
|
+
const name = videoFrameCacheName(ref, header.times.length, i);
|
|
2628
|
+
pinned.add(name);
|
|
2629
|
+
return path.join(dir, name);
|
|
2630
|
+
});
|
|
2631
|
+
const cached = await Promise.all(files.map((file) => cacheOrTouch(file, noBytes)));
|
|
2632
|
+
if (cached.every(Boolean)) {
|
|
2633
|
+
return { ...header, files };
|
|
2634
|
+
}
|
|
2635
|
+
}
|
|
2636
|
+
const bytes = await this._readBlob(digest.url, signal);
|
|
2637
|
+
if (!bytes) {
|
|
2638
|
+
return null;
|
|
2639
|
+
}
|
|
2640
|
+
const target = { dir, pinned, digestBlobId: digest.blobId };
|
|
2641
|
+
try {
|
|
2642
|
+
return await this._cacheVideo(ref, unpackVideoDigest(bytes), target);
|
|
2643
|
+
}
|
|
2644
|
+
catch (err) {
|
|
2645
|
+
this._log?.warn('assistant', `video digest unreadable — ${ref.fileName}`, err);
|
|
2646
|
+
return null;
|
|
2647
|
+
}
|
|
2648
|
+
}
|
|
2649
|
+
async _cacheVideo(ref, digest, { dir, pinned, digestBlobId }) {
|
|
2650
|
+
const { frames, ...header } = digest;
|
|
2651
|
+
const files = frames.map((_, i) => {
|
|
2652
|
+
const name = videoFrameCacheName(ref, frames.length, i);
|
|
2653
|
+
pinned.add(name);
|
|
2654
|
+
return path.join(dir, name);
|
|
2655
|
+
});
|
|
2656
|
+
await Promise.all(frames.map((frame, i) => cacheOrTouch(files[i], () => Promise.resolve(frame))));
|
|
2657
|
+
if (digestBlobId) {
|
|
2658
|
+
const headerName = videoHeaderCacheName(digestBlobId);
|
|
2659
|
+
pinned.add(headerName);
|
|
2660
|
+
await cacheOrTouch(path.join(dir, headerName), () => Promise.resolve(b4a.from(JSON.stringify(header))));
|
|
2661
|
+
}
|
|
2662
|
+
return { ...header, files };
|
|
2663
|
+
}
|
|
2664
|
+
async _sampleVideo(video, probe, hydration) {
|
|
2665
|
+
const { ref, budget, signal } = hydration;
|
|
2666
|
+
const started = Date.now();
|
|
2667
|
+
const count = videoFrameCount(probe, budget);
|
|
2668
|
+
const times = frameTimes(probe.duration, count);
|
|
2669
|
+
const frames = await sampleFrames(video.file, { times, maxEdge: VIDEO_FRAME_EDGE, probe });
|
|
2670
|
+
const sampleMs = Date.now() - started;
|
|
2671
|
+
const heard = probe.hasAudio
|
|
2672
|
+
? await this._hearVideo(ref, () => Promise.resolve(video), signal)
|
|
2673
|
+
: { audio: 'none' };
|
|
2674
|
+
const digest = {
|
|
2675
|
+
duration: probe.duration,
|
|
2676
|
+
times: frames.map((frame) => frame.time),
|
|
2677
|
+
...heard,
|
|
2678
|
+
frames: frames.map((frame) => frame.jpeg)
|
|
2679
|
+
};
|
|
2680
|
+
if (!hydration.ref.digest) {
|
|
2681
|
+
hydration.sampled.push({ ref, digest });
|
|
2682
|
+
}
|
|
2683
|
+
this._log?.info('assistant', 'video frames', {
|
|
2684
|
+
fileName: ref.fileName,
|
|
2685
|
+
duration: probe.duration,
|
|
2686
|
+
frames: frames.length,
|
|
2687
|
+
budget: count,
|
|
2688
|
+
ctxSize: budget.ctxSize ?? null,
|
|
2689
|
+
cacheTokens: budget.cacheTokens ?? null,
|
|
2690
|
+
audio: heard.audio,
|
|
2691
|
+
sampleMs
|
|
2692
|
+
});
|
|
2693
|
+
return this._cacheVideo(ref, digest, hydration);
|
|
2694
|
+
}
|
|
2695
|
+
async _storeVideoDigests(sampled, target) {
|
|
2696
|
+
const dir = this._imageCacheDir;
|
|
2697
|
+
const pinned = this._runImageCacheNames.get(target.runId) ?? new Set();
|
|
2698
|
+
for (const { ref, digest } of sampled) {
|
|
2699
|
+
try {
|
|
2700
|
+
const blobId = await this._uploadVideoDigest(ref, digest, target);
|
|
2701
|
+
await this._cacheVideo(ref, digest, { dir, pinned, digestBlobId: blobId });
|
|
2702
|
+
}
|
|
2703
|
+
catch (err) {
|
|
2704
|
+
this._log?.warn('assistant', `video digest not stored — ${ref.fileName}`, err);
|
|
2705
|
+
}
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
async _uploadVideoDigest(ref, digest, target) {
|
|
2709
|
+
const attachment = await this._uploadGenerated({
|
|
2710
|
+
fileName: videoDigestFileName(ref.fileName),
|
|
2711
|
+
mimeType: VIDEO_DIGEST_MIME,
|
|
2712
|
+
bytes: packVideoDigest(digest),
|
|
2713
|
+
raw: true
|
|
2714
|
+
}, { ...target, sourceId: ref.id });
|
|
2715
|
+
return attachment.blobId;
|
|
2716
|
+
}
|
|
2717
|
+
async _openVideo({ dir, ref, pinned, signal }) {
|
|
2718
|
+
const bytes = await this._readBlob(ref.url, signal);
|
|
2719
|
+
if (!bytes) {
|
|
2720
|
+
return null;
|
|
2721
|
+
}
|
|
2722
|
+
const name = `${imageCacheName(ref)}.source`;
|
|
2723
|
+
pinned.add(name);
|
|
2724
|
+
const file = path.join(dir, name);
|
|
2725
|
+
await fs.promises.writeFile(file, bytes, { mode: 0o600 });
|
|
2726
|
+
return { bytes, file };
|
|
2727
|
+
}
|
|
2728
|
+
async _hearVideo(ref, open, signal) {
|
|
2729
|
+
const cached = this._audioTranscripts.get(ref.blobId);
|
|
2730
|
+
if (cached !== undefined) {
|
|
2731
|
+
return cached ? { audio: 'speech', transcript: cached } : { audio: 'silent' };
|
|
2732
|
+
}
|
|
2733
|
+
if (!this._harness.transcribe) {
|
|
2734
|
+
return { audio: 'unheard' };
|
|
2735
|
+
}
|
|
2736
|
+
try {
|
|
2737
|
+
const video = await open();
|
|
2738
|
+
if (!video) {
|
|
2739
|
+
return { audio: 'unheard' };
|
|
2740
|
+
}
|
|
2741
|
+
const model = mediaModelNameFor('transcription', await this._persistedProfile());
|
|
2742
|
+
await this.ensureMediaModelUnpaused('transcription', model);
|
|
2743
|
+
await this._downloads?.waitUntilCached(model, signal);
|
|
2744
|
+
const transcript = (await this._harness.transcribe({ model: '', audio: b4a.toBuffer(video.bytes), signal })).trim();
|
|
2745
|
+
this._audioTranscripts.set(ref.blobId, transcript);
|
|
2746
|
+
return transcript ? { audio: 'speech', transcript } : { audio: 'silent' };
|
|
2747
|
+
}
|
|
2748
|
+
catch (err) {
|
|
2749
|
+
if (signal.aborted) {
|
|
2750
|
+
throw err;
|
|
2751
|
+
}
|
|
2752
|
+
this._log?.warn('assistant', `video transcription failed — ${ref.fileName}`, err);
|
|
2753
|
+
return { audio: 'unheard' };
|
|
2754
|
+
}
|
|
2755
|
+
}
|
|
2407
2756
|
async _hydrateImages(messages, signal, runId) {
|
|
2408
2757
|
const unreadable = new Map();
|
|
2409
2758
|
const turns = messages.filter((turn) => turn.images?.length);
|
|
@@ -2633,6 +2982,16 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2633
2982
|
}
|
|
2634
2983
|
}
|
|
2635
2984
|
}
|
|
2985
|
+
async _contextReading({ chatId, meshId, folded }, ctxSize) {
|
|
2986
|
+
const last = folded ? {} : await this._lastRunMetrics(chatId, meshId);
|
|
2987
|
+
const window = ctxSize || last.ctxSize;
|
|
2988
|
+
if (!window) {
|
|
2989
|
+
return {};
|
|
2990
|
+
}
|
|
2991
|
+
return last.ctxSize === window
|
|
2992
|
+
? { ctxSize: window, cacheTokens: last.cacheTokens }
|
|
2993
|
+
: { ctxSize: window };
|
|
2994
|
+
}
|
|
2636
2995
|
async _lastRunMetrics(chatId, meshId) {
|
|
2637
2996
|
const deviceId = this._requireDeviceId();
|
|
2638
2997
|
const { chunks } = await meshScoped(this._engine, meshId)
|
|
@@ -166,6 +166,7 @@ async function forkedChunk(ctx, chunk) {
|
|
|
166
166
|
chatId: ctx.chatId,
|
|
167
167
|
runId: chunk.runId && forkedId(ctx, chunk.runId),
|
|
168
168
|
parentId: chunk.parentId && ctx.kept.has(chunk.parentId) ? forkedId(ctx, chunk.parentId) : null,
|
|
169
|
+
sourceId: chunk.sourceId && forkedId(ctx, chunk.sourceId),
|
|
169
170
|
compaction: chunk.compaction && {
|
|
170
171
|
...chunk.compaction,
|
|
171
172
|
throughId: forkedId(ctx, chunk.compaction.throughId)
|
|
@@ -177,7 +177,7 @@ function matchAttachments(chunks, needle, cap) {
|
|
|
177
177
|
if (matches.length >= cap)
|
|
178
178
|
break;
|
|
179
179
|
const attachment = chunk.attachment;
|
|
180
|
-
if (!attachment || (attachment.deletedAt ?? 0) > 0) {
|
|
180
|
+
if (!attachment || (attachment.deletedAt ?? 0) > 0 || chunk.sourceId) {
|
|
181
181
|
continue;
|
|
182
182
|
}
|
|
183
183
|
const found = locate(attachment.fileName, needle);
|
|
@@ -3,7 +3,7 @@ import type { QvacChunk, QvacUpdateChunkOperation } from '../../spec/mesh/hypers
|
|
|
3
3
|
import type { Mesh, MeshDispatcher } from './mesh.ts';
|
|
4
4
|
import { type Bounds } from './range.ts';
|
|
5
5
|
type ChunkPayload = Pick<QvacChunk, 'userMessage' | 'content' | 'thinking' | 'toolCall' | 'toolResult' | 'attachment' | 'approvalRequest' | 'runStatus' | 'runRequest' | 'sources' | 'compaction'>;
|
|
6
|
-
type AppendChunkInput = Pick<QvacChunk, 'chatId' | 'runId' | 'type' | 'originDeviceId' | 'parentId' | 'effect' | 'intent' | 'display' | 'rawCall' | 'capability' | 'capabilityArgs' | 'enhance'> & ChunkPayload & {
|
|
6
|
+
type AppendChunkInput = Pick<QvacChunk, 'chatId' | 'runId' | 'type' | 'originDeviceId' | 'parentId' | 'effect' | 'intent' | 'display' | 'rawCall' | 'capability' | 'capabilityArgs' | 'enhance' | 'sourceId'> & ChunkPayload & {
|
|
7
7
|
id?: string;
|
|
8
8
|
};
|
|
9
9
|
type UpdateChunkInput = Pick<QvacChunk, 'id' | 'runStatus' | 'attachment'>;
|
|
@@ -40,7 +40,7 @@ export declare function getChunk(store: Mesh, id: string): Promise<QvacChunk | n
|
|
|
40
40
|
export interface AppendedChunk {
|
|
41
41
|
readonly id: string;
|
|
42
42
|
}
|
|
43
|
-
export declare function appendChunk(store: Mesh, { id, chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, ...payload }: AppendChunkInput, into?: MeshDispatcher): Promise<AppendedChunk>;
|
|
43
|
+
export declare function appendChunk(store: Mesh, { id, chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, sourceId, ...payload }: AppendChunkInput, into?: MeshDispatcher): Promise<AppendedChunk>;
|
|
44
44
|
export declare function updateChunk(store: Mesh, { id, runStatus, attachment }: UpdateChunkInput): Promise<QvacChunk>;
|
|
45
45
|
export declare function updateOp(chunk: QvacChunk, { id, runStatus, attachment }: UpdateChunkInput): QvacUpdateChunkOperation;
|
|
46
46
|
export interface ResolveApprovalInput {
|
|
@@ -118,7 +118,7 @@ function assertPayloadMatchesType(type, payload) {
|
|
|
118
118
|
throw new CoreError('CHUNK_PAYLOAD_INVALID', [`${type} carries ${field} and nothing else`]);
|
|
119
119
|
}
|
|
120
120
|
}
|
|
121
|
-
export async function appendChunk(store, { id = randomId(), chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, ...payload }, into = store) {
|
|
121
|
+
export async function appendChunk(store, { id = randomId(), chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, sourceId, ...payload }, into = store) {
|
|
122
122
|
assertPayloadMatchesType(type, payload);
|
|
123
123
|
if (type === 'sources' && !runId) {
|
|
124
124
|
throw new CoreError('CHUNK_PAYLOAD_INVALID', ['a sources chunk requires runId']);
|
|
@@ -129,6 +129,9 @@ export async function appendChunk(store, { id = randomId(), chatId, runId, type,
|
|
|
129
129
|
if (capability && type !== 'run-request') {
|
|
130
130
|
throw new CoreError('CHUNK_PAYLOAD_INVALID', ['only a run-request carries a capability']);
|
|
131
131
|
}
|
|
132
|
+
if (sourceId && type !== 'attachment') {
|
|
133
|
+
throw new CoreError('CHUNK_PAYLOAD_INVALID', ['only an attachment is derived from another']);
|
|
134
|
+
}
|
|
132
135
|
if (type === 'user-message' && !store.ourMemberId) {
|
|
133
136
|
await store.refreshOurMember();
|
|
134
137
|
}
|
|
@@ -154,6 +157,7 @@ export async function appendChunk(store, { id = randomId(), chatId, runId, type,
|
|
|
154
157
|
capability,
|
|
155
158
|
capabilityArgs,
|
|
156
159
|
enhance,
|
|
160
|
+
sourceId,
|
|
157
161
|
createdAt: Date.now(),
|
|
158
162
|
senderMemberId: type === 'user-message' && store.ourDeviceId && b4a.equals(store.ourDeviceId, originDeviceId)
|
|
159
163
|
? store.ourMemberId || undefined
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { isAudioAttachment } from './audio-attachment.js';
|
|
2
2
|
import { classifyRagDocument, RAG_DOCUMENT_KIND } from './files/rag/index.js';
|
|
3
3
|
import { IMAGE_MIMES } from './image/normalize.js';
|
|
4
|
+
import { isVideoAttachment } from './video/attachment.js';
|
|
4
5
|
export function isDocumentAttachment(ref) {
|
|
5
6
|
if (IMAGE_MIMES.has(ref.mimeType.toLowerCase()))
|
|
6
7
|
return false;
|
|
8
|
+
if (isVideoAttachment(ref.fileName, ref.mimeType))
|
|
9
|
+
return false;
|
|
7
10
|
return !isAudioAttachment(ref.fileName, ref.mimeType);
|
|
8
11
|
}
|
|
9
12
|
export function isPdfAttachment(ref) {
|
|
@@ -4,7 +4,7 @@ import { attachmentFiles } from './attachments.js';
|
|
|
4
4
|
import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, reclaimPurgedFiles, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId } from './records/index.js';
|
|
5
5
|
import { knowledgeFiles } from './knowledge.js';
|
|
6
6
|
import { randomId } from '../id.js';
|
|
7
|
-
import { getChunk, listChunks } from '../chunk.js';
|
|
7
|
+
import { findChunksOfType, getChunk, listChunks } from '../chunk.js';
|
|
8
8
|
import { settleFiles, isClaimLive, listIndexingByChat, FileState, FileProcessing } from './processing/index.js';
|
|
9
9
|
import { DeadlineScheduler } from '../../deadline.js';
|
|
10
10
|
import { liveDeviceForWriter } from '../contract/activity.js';
|
|
@@ -14,7 +14,7 @@ import { RagProcessor } from './rag/index.js';
|
|
|
14
14
|
import { PreviewProcessor, PREVIEW_VERSION, previewId } from './preview/index.js';
|
|
15
15
|
import { CoreError } from '../../errors.js';
|
|
16
16
|
import { errorMessage } from '../../guards.js';
|
|
17
|
-
import { ACTIVE_KNOWLEDGE_BY_CHAT, AGENTS, CHATS,
|
|
17
|
+
import { ACTIVE_KNOWLEDGE_BY_CHAT, AGENTS, CHATS, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE } from '../collections.js';
|
|
18
18
|
export { fullTextBudgetChars, fullTextHits, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from './rag/index.js';
|
|
19
19
|
export { isKnowledgeRoutedAttachment } from './records/index.js';
|
|
20
20
|
export { assertRemovableAt } from './records/index.js';
|
|
@@ -457,13 +457,13 @@ export class Files extends EventEmitter {
|
|
|
457
457
|
}
|
|
458
458
|
const agent = await snap.get(AGENTS, { id: chat.agentId });
|
|
459
459
|
const knowledge = await snap.find(ACTIVE_KNOWLEDGE_BY_CHAT, range).toArray();
|
|
460
|
-
const chunks = await snap
|
|
460
|
+
const chunks = await findChunksOfType(snap, { chatId, type: 'attachment' });
|
|
461
461
|
const statuses = await snap.find(INDEXING_STATUS_BY_CHAT, range).toArray();
|
|
462
462
|
const sources = new Map(knowledge.map((row) => [row.id, row]));
|
|
463
463
|
const ragIds = new Set(sources.keys());
|
|
464
464
|
for (const chunk of chunks) {
|
|
465
465
|
const file = chunk.attachment;
|
|
466
|
-
if (!file || file.deletedAt || sources.has(file.id)) {
|
|
466
|
+
if (!file || file.deletedAt || chunk.sourceId || sources.has(file.id)) {
|
|
467
467
|
continue;
|
|
468
468
|
}
|
|
469
469
|
sources.set(file.id, { ...file, chatId, updatedAt: file.createdAt });
|