@qvac/core 0.11.2 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -1
- package/dist-lib/lib/assistant/history.js +1 -1
- package/dist-lib/lib/assistant/message.d.ts +4 -1
- package/dist-lib/lib/assistant/pdf-budget.d.ts +10 -2
- package/dist-lib/lib/assistant/pdf-budget.js +10 -11
- package/dist-lib/lib/assistant/pdf-pages.d.ts +18 -0
- package/dist-lib/lib/assistant/pdf-pages.js +75 -0
- package/dist-lib/lib/assistant.d.ts +5 -5
- package/dist-lib/lib/assistant.js +144 -122
- package/dist-lib/lib/core/document-attachment.d.ts +6 -3
- package/dist-lib/lib/core/document-attachment.js +10 -5
- package/dist-lib/lib/core/files/index.d.ts +1 -1
- package/dist-lib/lib/core/files/index.js +1 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/index.js +1 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +3 -0
- package/dist-lib/lib/core/mesh.d.ts +3 -0
- package/dist-lib/lib/core/mesh.js +55 -17
- package/dist-lib/lib/core/migrations/file-index.d.ts +1 -1
- package/dist-lib/lib/core/migrations/file-index.js +6 -2
- package/dist-lib/lib/core/migrations/index.d.ts +2 -1
- package/dist-lib/lib/core/migrations/index.js +65 -8
- package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
- package/dist-lib/lib/core/pdf/figure.js +64 -0
- package/dist-lib/lib/core/pdf/rasterize.d.ts +10 -1
- package/dist-lib/lib/core/pdf/rasterize.js +72 -22
- package/dist-lib/lib/core.js +21 -3
- package/package.json +2 -2
|
@@ -21,8 +21,8 @@ import { meshScoped } from './mesh-scoped-engine.js';
|
|
|
21
21
|
import { buildTitlePrompt, deriveChatTitle, isDefaultChatTitle, parseTitleResponse } from './chat-title.js';
|
|
22
22
|
import { matchApprovalRule } from './core/approval-rules.js';
|
|
23
23
|
import { isAudioAttachment } from './core/audio-attachment.js';
|
|
24
|
-
import { attachTurnDocumentIds, formatRagContext, chooseRagNotice, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, createRagSourceTagFilter, citedByPhraseOverlap, formatFullDocumentContext, isFullTextHit, isTruncatedFullTextHit, sourcesFromHits, WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from './core/files/index.js';
|
|
25
|
-
import { annotateDocumentAttachments, documentUnreadNotice, documentUnreadReason, pdfPagesCutNotice,
|
|
24
|
+
import { attachTurnDocumentIds, formatRagContext, chooseRagNotice, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, createRagSourceTagFilter, citedByPhraseOverlap, formatFullDocumentContext, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, sourcesFromHits, WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from './core/files/index.js';
|
|
25
|
+
import { annotateDocumentAttachments, documentUnreadNotice, documentUnreadReason, PDF_PAGES_SOURCE_LINE, pdfPagesCutNotice, pdfPagesShownNotice } from './core/document-attachment.js';
|
|
26
26
|
import { imageCacheKey, imageCacheName, sweepImageCache } from './core/image/cache-sweep.js';
|
|
27
27
|
import { IMAGE_MIMES, normalizeImageForModel } from './core/image/normalize.js';
|
|
28
28
|
import { createPdfRasterizer } from './core/pdf/rasterize.js';
|
|
@@ -51,7 +51,8 @@ import { frameTimes, probeVideo, sampleFrames } from './core/video/frames.js';
|
|
|
51
51
|
import { packVideoDigest, parseVideoDigestHeader, unpackVideoDigest, VIDEO_DIGEST_MIME, videoDigestFileName } from './core/video/digest.js';
|
|
52
52
|
import { VIDEO_FRAME_EDGE, videoFrameCount } from './assistant/video-budget.js';
|
|
53
53
|
import { drainBody, raceInactivity } from './assistant/fetch-body.js';
|
|
54
|
-
import {
|
|
54
|
+
import { pdfImageRoom } from './assistant/pdf-budget.js';
|
|
55
|
+
import { packPdfSelection, parsePdfSelection, PDF_PAGES_MIME, pdfSelectionFileName, selectPdfPages } from './assistant/pdf-pages.js';
|
|
55
56
|
import { effectiveCtxSize } from './api/effective-ctx-size.js';
|
|
56
57
|
import { GENERIC_ACKNOWLEDGEMENTS, HISTORY_TYPES, SKILL_TOOL, TERMINAL_STATES, couldNameSenders, historyFromChunks, inputAvailableToRun, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds } from './assistant/history.js';
|
|
57
58
|
import { errorMessage, isRecord } from './guards.js';
|
|
@@ -114,19 +115,16 @@ const RunOccupancySchema = z.object({
|
|
|
114
115
|
export const CAPTURE_INFERENCE_CALLS_KEY = 'inspector.captureInferenceCalls';
|
|
115
116
|
export const CAPTURE_ON = 'on';
|
|
116
117
|
export const CAPTURE_OFF = 'off';
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
return undefined;
|
|
118
|
+
const NO_PDF_PAGES = { selection: { pageCount: 0, shots: [] }, names: [], labels: [] };
|
|
119
|
+
function pdfNotices(fileName, pages, wholeText) {
|
|
120
|
+
const { cutAt, pageCount } = pages.selection;
|
|
121
|
+
const shown = pages.names.length;
|
|
122
|
+
return [
|
|
123
|
+
...(shown ? [pdfPagesShownNotice(fileName, pages.labels)] : []),
|
|
124
|
+
...(cutAt === undefined
|
|
125
|
+
? []
|
|
126
|
+
: [pdfPagesCutNotice({ fileName, cutAt, pageCount, shown, wholeText })])
|
|
127
|
+
];
|
|
130
128
|
}
|
|
131
129
|
function attachPages(turn, files) {
|
|
132
130
|
if (!files.length)
|
|
@@ -154,8 +152,22 @@ function videoAudioBlock(fileName, { audio, transcript }) {
|
|
|
154
152
|
}
|
|
155
153
|
return unheardAudioNotice(`The audio of the attached video "${fileName}"`, 'transcription failed');
|
|
156
154
|
}
|
|
157
|
-
function
|
|
158
|
-
|
|
155
|
+
async function cachedPdfPages(selection, { dir, pdf, pinned }) {
|
|
156
|
+
const names = selection.shots.map((shot) => pdfShotCacheName(pdf, shot));
|
|
157
|
+
for (const name of names) {
|
|
158
|
+
pinned.add(name);
|
|
159
|
+
}
|
|
160
|
+
const files = names.map((name) => path.join(dir, name));
|
|
161
|
+
const cached = await Promise.all(files.map((file) => cacheOrTouch(file, noBytes)));
|
|
162
|
+
return cached.every(Boolean)
|
|
163
|
+
? { selection, names, labels: selection.shots.map((shot) => shot.label) }
|
|
164
|
+
: null;
|
|
165
|
+
}
|
|
166
|
+
function pdfShotCacheName(pdf, shot) {
|
|
167
|
+
const box = shot.clip
|
|
168
|
+
? `.c${[shot.clip.x, shot.clip.y, shot.clip.width, shot.clip.height].map(Math.round).join('_')}`
|
|
169
|
+
: '';
|
|
170
|
+
return imageCacheName({ blobId: pdf.blobId, fileName: `${pdf.fileName}.p${shot.page}${box}.jpg` });
|
|
159
171
|
}
|
|
160
172
|
async function cacheOrTouch(file, produce) {
|
|
161
173
|
const cached = await fs.promises.stat(file).catch(noop);
|
|
@@ -268,7 +280,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
268
280
|
_lifecycle;
|
|
269
281
|
_audioTranscripts;
|
|
270
282
|
_videoProbes;
|
|
271
|
-
|
|
283
|
+
_pdfSelections;
|
|
272
284
|
_chats;
|
|
273
285
|
_runImageCacheNames;
|
|
274
286
|
_reconciledCompactions;
|
|
@@ -328,7 +340,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
328
340
|
this._lifecycle = new AbortController();
|
|
329
341
|
this._audioTranscripts = new Map();
|
|
330
342
|
this._videoProbes = new Map();
|
|
331
|
-
this.
|
|
343
|
+
this._pdfSelections = new Map();
|
|
332
344
|
this._chats = new Map();
|
|
333
345
|
this._runImageCacheNames = new Map();
|
|
334
346
|
this._reconciledCompactions = new Map();
|
|
@@ -688,7 +700,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
688
700
|
this._lastCapabilities.set(meshId, key);
|
|
689
701
|
}
|
|
690
702
|
catch (err) {
|
|
691
|
-
if (isCoreError(err, 'MESH_NOT_FOUND')) {
|
|
703
|
+
if (isCoreError(err, 'MESH_NOT_FOUND') || isCoreError(err, 'MESH_STILL_JOINING')) {
|
|
692
704
|
return;
|
|
693
705
|
}
|
|
694
706
|
this._log?.error('assistant', 'saveDeviceCapabilities failed', err);
|
|
@@ -1808,7 +1820,8 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1808
1820
|
const seesImages = sendsFiles && carriesMedia && (await this._seesImages(model));
|
|
1809
1821
|
let unreadableImages = new Map();
|
|
1810
1822
|
const sampled = [];
|
|
1811
|
-
let pdfPages = { rendered: new Set(),
|
|
1823
|
+
let pdfPages = { rendered: new Set(), notices: [] };
|
|
1824
|
+
const selectedPdfs = [];
|
|
1812
1825
|
if (!seesImages) {
|
|
1813
1826
|
const dropped = messages.reduce((count, turn) => count + (turn.images?.length ?? 0), 0);
|
|
1814
1827
|
if (dropped && (this._harness.supportsImages ?? this._info.supportsImages)) {
|
|
@@ -2009,7 +2022,12 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2009
2022
|
this._knowledgeCaps.delete(chatId);
|
|
2010
2023
|
const fitted = knowledge && cap ? { ...knowledge, hits: knowledge.hits.slice(0, cap) } : knowledge;
|
|
2011
2024
|
const injected = fitted
|
|
2012
|
-
? await this._injectKnowledge(fitted, {
|
|
2025
|
+
? await this._injectKnowledge(fitted, {
|
|
2026
|
+
transcript,
|
|
2027
|
+
sourceTagFilter,
|
|
2028
|
+
retrievedRecords,
|
|
2029
|
+
pictured: seesImages && messages.some((turn) => turn.pdfs?.length)
|
|
2030
|
+
})
|
|
2013
2031
|
: { documentsInjected: false, sourcesWritten: false, ragInjected: false };
|
|
2014
2032
|
injectedHits = injected.ragInjected && fitted ? fitted.hits.length : 0;
|
|
2015
2033
|
const hasAudio = messages.some((turn) => turn.audioAttachments?.length || turn.videos?.length);
|
|
@@ -2032,14 +2050,22 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2032
2050
|
}
|
|
2033
2051
|
backfillImageOnlyTurns(messages);
|
|
2034
2052
|
if (seesImages) {
|
|
2053
|
+
const hits = injected.documentsInjected && fitted ? fitted.hits : [];
|
|
2035
2054
|
pdfPages = await this._hydratePdfPages(messages, {
|
|
2036
2055
|
signal: controller.signal,
|
|
2037
2056
|
runId,
|
|
2038
|
-
window: { ctxSize: configuredAgent.ctxSize, modelName: model }
|
|
2057
|
+
window: { ctxSize: configuredAgent.ctxSize, modelName: model },
|
|
2058
|
+
reading: { chatId, meshId, folded },
|
|
2059
|
+
textChars: hits.reduce((chars, hit) => chars + hit.content.length, 0),
|
|
2060
|
+
wholeIds: new Set(hits
|
|
2061
|
+
.filter((hit) => !isTruncatedFullTextHit(hit))
|
|
2062
|
+
.map(fullTextAttachmentId)
|
|
2063
|
+
.filter((id) => id !== null)),
|
|
2064
|
+
selected: selectedPdfs
|
|
2039
2065
|
});
|
|
2040
2066
|
}
|
|
2041
2067
|
annotateDocumentAttachments(messages);
|
|
2042
|
-
for (const { turn, notice } of pdfPages.
|
|
2068
|
+
for (const { turn, notice } of pdfPages.notices) {
|
|
2043
2069
|
turn.content = `${turn.content}\n${notice}`;
|
|
2044
2070
|
}
|
|
2045
2071
|
const unreadIds = attachTurnIds.filter((id) => !pdfPages.rendered.has(id));
|
|
@@ -2247,6 +2273,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2247
2273
|
});
|
|
2248
2274
|
titleOutcome = 'completed';
|
|
2249
2275
|
await this._storeVideoDigests(sampled, { meshId, chatId, runId, deviceId });
|
|
2276
|
+
await this._storePdfSelections(selectedPdfs, { meshId, chatId, runId, deviceId });
|
|
2250
2277
|
}
|
|
2251
2278
|
catch (err) {
|
|
2252
2279
|
const outcome = controller.signal.aborted
|
|
@@ -2885,39 +2912,41 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2885
2912
|
});
|
|
2886
2913
|
return file;
|
|
2887
2914
|
}
|
|
2888
|
-
async _hydratePdfPages(messages, { signal, runId, window }) {
|
|
2915
|
+
async _hydratePdfPages(messages, { signal, runId, window, reading, textChars, wholeIds, selected }) {
|
|
2889
2916
|
const rendered = new Set();
|
|
2890
|
-
const
|
|
2917
|
+
const notices = [];
|
|
2891
2918
|
const turns = messages.filter((turn) => turn.pdfs?.length);
|
|
2892
2919
|
if (!turns.length) {
|
|
2893
|
-
return { rendered,
|
|
2920
|
+
return { rendered, notices };
|
|
2894
2921
|
}
|
|
2895
2922
|
const dir = this._imageCacheDir;
|
|
2896
2923
|
await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
|
|
2897
2924
|
const pinned = this._runImageCacheNames.get(runId) ?? new Set();
|
|
2898
2925
|
this._runImageCacheNames.set(runId, pinned);
|
|
2899
2926
|
const latestUser = messages.findLast((turn) => turn.role === 'user');
|
|
2900
|
-
const
|
|
2927
|
+
const measured = await this._pdfReading(window, reading);
|
|
2901
2928
|
for (const turn of turns) {
|
|
2902
|
-
const
|
|
2903
|
-
|
|
2904
|
-
pinned,
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2929
|
+
const room = pdfImageRoom(measured, turn === latestUser ? textChars : 0);
|
|
2930
|
+
for (const pdf of turn.pdfs ?? []) {
|
|
2931
|
+
const pages = await this._tryHydratePdf({ dir, pdf, pinned, room, signal, selected });
|
|
2932
|
+
attachPages(turn, pages.names.map((name) => path.join(dir, name)));
|
|
2933
|
+
if (pages.names.length && turn === latestUser) {
|
|
2934
|
+
rendered.add(pdf.id);
|
|
2935
|
+
}
|
|
2936
|
+
const whole = turn === latestUser && wholeIds.has(pdf.id);
|
|
2937
|
+
for (const notice of pdfNotices(pdf.fileName, pages, whole)) {
|
|
2938
|
+
notices.push({ turn, notice });
|
|
2911
2939
|
}
|
|
2912
2940
|
}
|
|
2913
|
-
|
|
2941
|
+
delete turn.pdfs;
|
|
2914
2942
|
}
|
|
2915
|
-
return { rendered,
|
|
2943
|
+
return { rendered, notices };
|
|
2916
2944
|
}
|
|
2917
|
-
async
|
|
2945
|
+
async _pdfReading({ ctxSize, modelName }, options) {
|
|
2918
2946
|
const custom = await this._customModel(modelName);
|
|
2919
2947
|
const window = effectiveCtxSize({ ctxSize, modelName }, { appCtxSize: 0, platform: this._platform, custom });
|
|
2920
|
-
|
|
2948
|
+
const { cacheTokens } = await this._contextReading(options, window);
|
|
2949
|
+
return { ctxSize: window, cacheTokens };
|
|
2921
2950
|
}
|
|
2922
2951
|
async _customModel(name) {
|
|
2923
2952
|
for await (const list of this._engine.customModelsWatch({})) {
|
|
@@ -2925,113 +2954,104 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2925
2954
|
}
|
|
2926
2955
|
return null;
|
|
2927
2956
|
}
|
|
2928
|
-
async
|
|
2929
|
-
const shown = [];
|
|
2930
|
-
const notices = [];
|
|
2931
|
-
let budget = maxPages;
|
|
2932
|
-
for (const pdf of turn.pdfs ?? []) {
|
|
2933
|
-
if (budget <= 0) {
|
|
2934
|
-
notices.push(pdfPagesSkippedNotice(pdf.fileName));
|
|
2935
|
-
continue;
|
|
2936
|
-
}
|
|
2937
|
-
const pages = await this._tryHydratePdf(dir, turn, { pdf, pinned, maxPages: budget, signal });
|
|
2938
|
-
budget -= pages.tried;
|
|
2939
|
-
if (pages.names.length) {
|
|
2940
|
-
shown.push(pdf.id);
|
|
2941
|
-
}
|
|
2942
|
-
if (pages.cut) {
|
|
2943
|
-
notices.push(pdfPagesCutNotice({ fileName: pdf.fileName, ...pages.cut }));
|
|
2944
|
-
}
|
|
2945
|
-
}
|
|
2946
|
-
delete turn.pdfs;
|
|
2947
|
-
return { shown, notices };
|
|
2948
|
-
}
|
|
2949
|
-
async _tryHydratePdf(dir, turn, opts) {
|
|
2957
|
+
async _tryHydratePdf(opts) {
|
|
2950
2958
|
try {
|
|
2951
|
-
return await this._hydratePdf(
|
|
2959
|
+
return await this._hydratePdf(opts);
|
|
2952
2960
|
}
|
|
2953
2961
|
catch (err) {
|
|
2954
2962
|
if (opts.signal.aborted) {
|
|
2955
2963
|
throw err;
|
|
2956
2964
|
}
|
|
2957
2965
|
this._log?.warn('assistant', `pdf rasterization failed — ${opts.pdf.fileName}`, err);
|
|
2958
|
-
return
|
|
2966
|
+
return NO_PDF_PAGES;
|
|
2959
2967
|
}
|
|
2960
2968
|
}
|
|
2961
|
-
async _hydratePdf(
|
|
2962
|
-
const
|
|
2963
|
-
const
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2969
|
+
async _hydratePdf(hydration) {
|
|
2970
|
+
const { dir, pdf, pinned, room, signal, selected } = hydration;
|
|
2971
|
+
const key = pdf.pages?.blobId ?? pdf.id;
|
|
2972
|
+
const remember = (selection) => {
|
|
2973
|
+
this._pdfSelections.set(key, selection);
|
|
2974
|
+
if (!pdf.pages) {
|
|
2975
|
+
selected.push({ ref: pdf, selection });
|
|
2967
2976
|
}
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2977
|
+
};
|
|
2978
|
+
const known = this._pdfSelections.get(key) ?? (await this._storedPdfSelection(pdf, signal));
|
|
2979
|
+
if (known) {
|
|
2980
|
+
remember(known);
|
|
2981
|
+
const cached = await cachedPdfPages(known, { dir, pdf, pinned });
|
|
2982
|
+
if (cached) {
|
|
2983
|
+
return cached;
|
|
2972
2984
|
}
|
|
2973
2985
|
}
|
|
2974
|
-
const pages = await this._renderPdf(dir, { pdf, pinned, maxPages, signal });
|
|
2975
|
-
if (!pages) {
|
|
2976
|
-
return { names: [], tried: 0 };
|
|
2977
|
-
}
|
|
2978
|
-
this._pdfPages.set(key, pages);
|
|
2979
|
-
attachPages(turn, pages.names.map((name) => path.join(dir, name)));
|
|
2980
|
-
return pages;
|
|
2981
|
-
}
|
|
2982
|
-
async _renderPdf(dir, { pdf, pinned, maxPages, signal }) {
|
|
2983
2986
|
const bytes = await this._readBlob(pdf.url, signal);
|
|
2984
2987
|
if (signal.aborted) {
|
|
2985
2988
|
throw new Error('hydration aborted');
|
|
2986
2989
|
}
|
|
2987
|
-
|
|
2988
|
-
return null;
|
|
2989
|
-
}
|
|
2990
|
-
const doc = await this._pdfRasterizer.open(bytes);
|
|
2990
|
+
const doc = bytes ? await this._pdfRasterizer.open(bytes) : null;
|
|
2991
2991
|
if (!doc) {
|
|
2992
|
-
return
|
|
2992
|
+
return NO_PDF_PAGES;
|
|
2993
2993
|
}
|
|
2994
2994
|
try {
|
|
2995
|
-
const
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
const pageCount = doc.pageCount();
|
|
2999
|
-
for (; page < pageCount && tried < maxPages; page++) {
|
|
3000
|
-
if (signal.aborted)
|
|
3001
|
-
throw new Error('hydration aborted');
|
|
3002
|
-
const outcome = await this._renderPdfPage(doc, { dir, pdf, page, pinned, signal });
|
|
3003
|
-
if (outcome.tried) {
|
|
3004
|
-
tried++;
|
|
3005
|
-
}
|
|
3006
|
-
if (outcome.name) {
|
|
3007
|
-
names.push(outcome.name);
|
|
3008
|
-
}
|
|
2995
|
+
const selection = known ?? (await selectPdfPages(doc, room));
|
|
2996
|
+
if (!known) {
|
|
2997
|
+
remember(selection);
|
|
3009
2998
|
}
|
|
3010
|
-
return
|
|
2999
|
+
return await this._renderPdfShots(doc, selection, hydration);
|
|
3011
3000
|
}
|
|
3012
3001
|
finally {
|
|
3013
3002
|
doc.close();
|
|
3014
3003
|
}
|
|
3015
3004
|
}
|
|
3016
|
-
async
|
|
3017
|
-
|
|
3005
|
+
async _storedPdfSelection(pdf, signal) {
|
|
3006
|
+
const bytes = pdf.pages ? await this._readBlob(pdf.pages.url, signal) : null;
|
|
3007
|
+
if (!bytes) {
|
|
3008
|
+
return null;
|
|
3009
|
+
}
|
|
3018
3010
|
try {
|
|
3019
|
-
|
|
3020
|
-
if (flags.hasText && !flags.hasImage) {
|
|
3021
|
-
return { tried };
|
|
3022
|
-
}
|
|
3023
|
-
tried = true;
|
|
3024
|
-
const name = pdfPageCacheName(pdf, page);
|
|
3025
|
-
pinned.add(name);
|
|
3026
|
-
const cached = await cacheOrTouch(path.join(dir, name), () => doc.renderPage(page));
|
|
3027
|
-
return { tried, name: cached ? name : undefined };
|
|
3011
|
+
return parsePdfSelection(bytes);
|
|
3028
3012
|
}
|
|
3029
3013
|
catch (err) {
|
|
3030
|
-
|
|
3031
|
-
|
|
3014
|
+
this._log?.warn('assistant', `pdf page choice unreadable — ${pdf.fileName}`, err);
|
|
3015
|
+
return null;
|
|
3016
|
+
}
|
|
3017
|
+
}
|
|
3018
|
+
async _renderPdfShots(doc, selection, { dir, pdf, pinned, signal }) {
|
|
3019
|
+
const names = [];
|
|
3020
|
+
const labels = [];
|
|
3021
|
+
for (const shot of selection.shots) {
|
|
3022
|
+
if (signal.aborted)
|
|
3023
|
+
throw new Error('hydration aborted');
|
|
3024
|
+
const name = pdfShotCacheName(pdf, shot);
|
|
3025
|
+
pinned.add(name);
|
|
3026
|
+
try {
|
|
3027
|
+
const file = path.join(dir, name);
|
|
3028
|
+
if (await cacheOrTouch(file, () => doc.renderPage(shot.page, shot.clip))) {
|
|
3029
|
+
names.push(name);
|
|
3030
|
+
labels.push(shot.label);
|
|
3031
|
+
}
|
|
3032
|
+
}
|
|
3033
|
+
catch (err) {
|
|
3034
|
+
if (signal.aborted) {
|
|
3035
|
+
throw err;
|
|
3036
|
+
}
|
|
3037
|
+
this._log?.warn('assistant', `pdf page skipped — ${pdf.fileName} p${shot.page}`, err);
|
|
3038
|
+
}
|
|
3039
|
+
}
|
|
3040
|
+
return { selection, names, labels };
|
|
3041
|
+
}
|
|
3042
|
+
async _storePdfSelections(selected, target) {
|
|
3043
|
+
for (const { ref, selection } of selected) {
|
|
3044
|
+
try {
|
|
3045
|
+
await this._uploadGenerated({
|
|
3046
|
+
fileName: pdfSelectionFileName(ref.fileName),
|
|
3047
|
+
mimeType: PDF_PAGES_MIME,
|
|
3048
|
+
bytes: packPdfSelection(selection),
|
|
3049
|
+
raw: true
|
|
3050
|
+
}, { ...target, sourceId: ref.id });
|
|
3051
|
+
}
|
|
3052
|
+
catch (err) {
|
|
3053
|
+
this._log?.warn('assistant', `pdf page choice not stored — ${ref.fileName}`, err);
|
|
3032
3054
|
}
|
|
3033
|
-
this._log?.warn('assistant', `pdf page skipped — ${pdf.fileName} p${page}`, err);
|
|
3034
|
-
return { tried };
|
|
3035
3055
|
}
|
|
3036
3056
|
}
|
|
3037
3057
|
async _compactIfNearFull(params) {
|
|
@@ -3706,7 +3726,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
3706
3726
|
};
|
|
3707
3727
|
}
|
|
3708
3728
|
}
|
|
3709
|
-
async function injectRetrievedKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords, injected }) {
|
|
3729
|
+
async function injectRetrievedKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords, pictured, injected }) {
|
|
3710
3730
|
const records = sourcesFromHits(knowledge.hits, RAG_SOURCE_BOUNDS);
|
|
3711
3731
|
const notice = chooseRagNotice({
|
|
3712
3732
|
errorMessage: knowledge.search.errorMessage,
|
|
@@ -3729,7 +3749,9 @@ async function injectRetrievedKnowledge(knowledge, { transcript, sourceTagFilter
|
|
|
3729
3749
|
});
|
|
3730
3750
|
injected.sourcesWritten = true;
|
|
3731
3751
|
if (records.length > 0) {
|
|
3732
|
-
|
|
3752
|
+
const framing = formatRagContext(knowledge.hits, knowledge.search.availableFiles ?? []);
|
|
3753
|
+
const pages = pictured ? `\n${PDF_PAGES_SOURCE_LINE}` : '';
|
|
3754
|
+
knowledge.lastUser.content = `${framing}${pages}\n\nThe user's message:\n${knowledge.lastUser.content}\n\n${RAG_CITATION_LINE}`;
|
|
3733
3755
|
injected.ragInjected = true;
|
|
3734
3756
|
injected.documentsInjected = true;
|
|
3735
3757
|
}
|
|
@@ -25,11 +25,14 @@ export declare function documentUnreadReason({ knowledgeOn, failedFiles, attache
|
|
|
25
25
|
export declare function documentUnreadNotice(reason: DocumentUnreadReason, plural: boolean): string;
|
|
26
26
|
export interface PdfPagesCutState {
|
|
27
27
|
readonly fileName: string;
|
|
28
|
-
readonly
|
|
28
|
+
readonly cutAt: number;
|
|
29
29
|
readonly pageCount: number;
|
|
30
|
+
readonly shown: number;
|
|
31
|
+
readonly wholeText: boolean;
|
|
30
32
|
}
|
|
31
|
-
export declare function pdfPagesCutNotice({ fileName,
|
|
32
|
-
export declare
|
|
33
|
+
export declare function pdfPagesCutNotice({ fileName, cutAt, pageCount, shown, wholeText }: PdfPagesCutState): string;
|
|
34
|
+
export declare const PDF_PAGES_SOURCE_LINE = "Page images from the PDFs in this chat are a source too: when an image shows what the excerpts do not, answer from the image.";
|
|
35
|
+
export declare function pdfPagesShownNotice(fileName: string, labels: ReadonlyArray<string>): string;
|
|
33
36
|
export interface AnnotatedMessage {
|
|
34
37
|
content: string;
|
|
35
38
|
documents?: DocumentAttachment[];
|
|
@@ -33,11 +33,16 @@ export function documentUnreadNotice(reason, plural) {
|
|
|
33
33
|
}[reason];
|
|
34
34
|
return `[The attached document${plural ? 's' : ''} could not be read for this reply: ${why}.]`;
|
|
35
35
|
}
|
|
36
|
-
export function pdfPagesCutNotice({ fileName,
|
|
37
|
-
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
|
|
36
|
+
export function pdfPagesCutNotice({ fileName, cutAt, pageCount, shown, wholeText }) {
|
|
37
|
+
const missing = shown
|
|
38
|
+
? `Some pictures in "${fileName}" (${pageCount} pages), the first on page ${cutAt + 1}, were not shown to you as images`
|
|
39
|
+
: `No picture in "${fileName}" (${pageCount} pages) was shown to you as an image`;
|
|
40
|
+
const text = wholeText ? '; its text is all above' : '';
|
|
41
|
+
return `[${missing}: they did not fit in the context window, and no tool can show them${text}. If asked about a picture you cannot see, say it could not be shown. Do not guess its content.]`;
|
|
42
|
+
}
|
|
43
|
+
export const PDF_PAGES_SOURCE_LINE = 'Page images from the PDFs in this chat are a source too: when an image shows what the excerpts do not, answer from the image.';
|
|
44
|
+
export function pdfPagesShownNotice(fileName, labels) {
|
|
45
|
+
return `[The images from "${fileName}" are, in order: ${labels.join('; ')}.]`;
|
|
41
46
|
}
|
|
42
47
|
export function annotateDocumentAttachments(messages) {
|
|
43
48
|
for (const turn of messages.filter((message) => message.documents?.length)) {
|
|
@@ -122,7 +122,7 @@ export declare class Files extends EventEmitter {
|
|
|
122
122
|
private _processorStatus;
|
|
123
123
|
close(): Promise<void>;
|
|
124
124
|
}
|
|
125
|
-
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.ts';
|
|
125
|
+
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.ts';
|
|
126
126
|
export type { IndexingEmbedder, RagExtractors, RagSource, VectorIndex } from './rag/index.ts';
|
|
127
127
|
export { attachTurnDocumentIds, groupAttachmentsByMessage } from './records/index.ts';
|
|
128
128
|
export { CoreBlobServer, PENDING_BLOB_REFERENCE, isPendingBlobReference, parseBlobReferenceCoreKey } from './store/index.ts';
|
|
@@ -658,6 +658,6 @@ function fileState(processor, opts) {
|
|
|
658
658
|
}
|
|
659
659
|
return { state: 'pending', reason: 'source unavailable on this device' };
|
|
660
660
|
}
|
|
661
|
-
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.js';
|
|
661
|
+
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.js';
|
|
662
662
|
export { attachTurnDocumentIds, groupAttachmentsByMessage } from './records/index.js';
|
|
663
663
|
export { CoreBlobServer, PENDING_BLOB_REFERENCE, isPendingBlobReference, parseBlobReferenceCoreKey } from './store/index.js';
|
|
@@ -31,6 +31,6 @@ export declare class RagProcessor implements FileProcessor {
|
|
|
31
31
|
export { MAX_KNOWLEDGE_BYTES } from './limits.ts';
|
|
32
32
|
export { canIndexChat } from './policy.ts';
|
|
33
33
|
export type { IndexingEmbedder, IndexingProgress } from './produce.ts';
|
|
34
|
-
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.ts';
|
|
34
|
+
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.ts';
|
|
35
35
|
export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, retrieveRagKnowledge, searchRagChunks, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
|
|
36
36
|
export type { RagArtifactPayload, RagDocumentRef, RagExtractors, RagSource, VectorHit, VectorIndex } from 'rag-extract';
|
|
@@ -70,5 +70,5 @@ export class RagProcessor {
|
|
|
70
70
|
}
|
|
71
71
|
export { MAX_KNOWLEDGE_BYTES } from './limits.js';
|
|
72
72
|
export { canIndexChat } from './policy.js';
|
|
73
|
-
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.js';
|
|
73
|
+
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.js';
|
|
74
74
|
export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, retrieveRagKnowledge, searchRagChunks, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
|
|
@@ -7,6 +7,7 @@ export interface IdentifiedHit {
|
|
|
7
7
|
readonly id: string;
|
|
8
8
|
}
|
|
9
9
|
export declare function isFullTextHit(hit: IdentifiedHit): boolean;
|
|
10
|
+
export declare function fullTextAttachmentId(hit: IdentifiedHit): string | null;
|
|
10
11
|
export type FullTextCoverage = {
|
|
11
12
|
readonly unit: 'pages';
|
|
12
13
|
readonly included: number;
|
|
@@ -21,6 +21,9 @@ const FULL_TEXT_HIT_PREFIX = 'full-text-';
|
|
|
21
21
|
export function isFullTextHit(hit) {
|
|
22
22
|
return hit.id.startsWith(FULL_TEXT_HIT_PREFIX);
|
|
23
23
|
}
|
|
24
|
+
export function fullTextAttachmentId(hit) {
|
|
25
|
+
return isFullTextHit(hit) ? hit.id.slice(FULL_TEXT_HIT_PREFIX.length) : null;
|
|
26
|
+
}
|
|
24
27
|
const FullTextExtrasSchema = z.object({
|
|
25
28
|
truncated: z.boolean().optional().catch(undefined),
|
|
26
29
|
coverage: z
|
|
@@ -45,6 +45,7 @@ export interface MeshOptions {
|
|
|
45
45
|
slowWriteMs?: number;
|
|
46
46
|
slowDrainMs?: number;
|
|
47
47
|
}
|
|
48
|
+
export declare const SCHEMA_VERSION: number;
|
|
48
49
|
export declare function meshPublicKeyFromSeed(seed: Buffer): Buffer;
|
|
49
50
|
export interface TrustScope {
|
|
50
51
|
readonly ourMemberId: Buffer | null;
|
|
@@ -178,6 +179,8 @@ export declare class Mesh extends ReadyResource {
|
|
|
178
179
|
private _watchStall;
|
|
179
180
|
private _endStall;
|
|
180
181
|
private _reportStall;
|
|
182
|
+
private _fastForwardHeld;
|
|
183
|
+
private _writersHeld;
|
|
181
184
|
private _coresNotHeld;
|
|
182
185
|
private _coreHeld;
|
|
183
186
|
batch(): MeshBatch;
|