@qvac/core 0.11.2 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -1
- package/dist-lib/lib/assistant/history.js +1 -1
- package/dist-lib/lib/assistant/message.d.ts +4 -1
- package/dist-lib/lib/assistant/pdf-budget.d.ts +10 -2
- package/dist-lib/lib/assistant/pdf-budget.js +10 -11
- package/dist-lib/lib/assistant/pdf-pages.d.ts +18 -0
- package/dist-lib/lib/assistant/pdf-pages.js +75 -0
- package/dist-lib/lib/assistant.d.ts +5 -5
- package/dist-lib/lib/assistant.js +143 -121
- package/dist-lib/lib/core/document-attachment.d.ts +6 -3
- package/dist-lib/lib/core/document-attachment.js +10 -5
- package/dist-lib/lib/core/files/index.d.ts +1 -1
- package/dist-lib/lib/core/files/index.js +1 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/index.js +1 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +3 -0
- package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
- package/dist-lib/lib/core/pdf/figure.js +64 -0
- package/dist-lib/lib/core/pdf/rasterize.d.ts +10 -1
- package/dist-lib/lib/core/pdf/rasterize.js +72 -22
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -13,6 +13,24 @@ release.
|
|
|
13
13
|
|
|
14
14
|
## [Unreleased]
|
|
15
15
|
|
|
16
|
+
## [0.11.3] - 2026-10-28
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
|
|
20
|
+
- The first run to complete stores which pictures of a PDF it sent, as an attachment chunk whose
|
|
21
|
+
`sourceId` is the PDF's attachment id, so later turns and other devices send the same ones. Do
|
|
22
|
+
not render an attachment chunk that carries a `sourceId`. (#640)
|
|
23
|
+
|
|
24
|
+
### Changed
|
|
25
|
+
|
|
26
|
+
- A PDF's pictures reach a vision model cut to each figure and its caption, named by page and
|
|
27
|
+
figure, instead of as whole pages, and white pages are no longer sent. The PDF's text and its
|
|
28
|
+
pictures now share half the context window, less what the chat already holds, where each took
|
|
29
|
+
half before: a PDF whose whole text fit filled the window and compacted on the next message.
|
|
30
|
+
(#640)
|
|
31
|
+
- `bare-pdfium` is required at `^1.8.0`, for `imageBounds` and `pageBox`; a host that packages the
|
|
32
|
+
addon ships that version. (#640)
|
|
33
|
+
|
|
16
34
|
## [0.11.2] - 2026-10-28
|
|
17
35
|
|
|
18
36
|
### Added
|
|
@@ -2070,7 +2088,8 @@ Published under the `release-0.1` dist-tag.
|
|
|
2070
2088
|
await assistant.ready()
|
|
2071
2089
|
```
|
|
2072
2090
|
|
|
2073
|
-
[Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.11.
|
|
2091
|
+
[Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.11.3...HEAD
|
|
2092
|
+
[0.11.3]: https://github.com/tetherto/qvac-core/compare/v0.11.2...v0.11.3
|
|
2074
2093
|
[0.11.2]: https://github.com/tetherto/qvac-core/compare/v0.11.1...v0.11.2
|
|
2075
2094
|
[0.11.1]: https://github.com/tetherto/qvac-core/compare/v0.11.0...v0.11.1
|
|
2076
2095
|
[0.11.0]: https://github.com/tetherto/qvac-core/compare/v0.10.4...v0.11.0
|
|
@@ -179,7 +179,7 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
|
|
|
179
179
|
turn.documents = documents;
|
|
180
180
|
}
|
|
181
181
|
if (pdfs.length) {
|
|
182
|
-
turn.pdfs = pdfs;
|
|
182
|
+
turn.pdfs = pdfs.map((ref) => ({ ...ref, pages: digests.get(ref.id) }));
|
|
183
183
|
}
|
|
184
184
|
history.push(turn);
|
|
185
185
|
continue;
|
|
@@ -12,7 +12,7 @@ export interface ChatMessage {
|
|
|
12
12
|
audioAttachments?: AttachmentRef[];
|
|
13
13
|
videos?: VideoRef[];
|
|
14
14
|
documents?: DocumentAttachment[];
|
|
15
|
-
pdfs?:
|
|
15
|
+
pdfs?: PdfRef[];
|
|
16
16
|
}
|
|
17
17
|
export declare const IMAGE_ONLY_USER_PROMPT = "The user sent one or more images without any accompanying text. Describe what the images show and respond to anything notable in them.";
|
|
18
18
|
export declare const DOCUMENT_ONLY_USER_PROMPT = "The user sent one or more documents without any accompanying text. Summarize the attached documents.";
|
|
@@ -50,6 +50,9 @@ export interface AttachmentRef {
|
|
|
50
50
|
export interface VideoRef extends AttachmentRef {
|
|
51
51
|
digest?: AttachmentRef;
|
|
52
52
|
}
|
|
53
|
+
export interface PdfRef extends AttachmentRef {
|
|
54
|
+
pages?: AttachmentRef;
|
|
55
|
+
}
|
|
53
56
|
export interface AttachInput {
|
|
54
57
|
fileName: string;
|
|
55
58
|
mimeType: string;
|
|
@@ -1,3 +1,11 @@
|
|
|
1
|
-
export declare const MIN_PDF_PAGE_IMAGES = 2;
|
|
2
1
|
export declare const MAX_PDF_PAGE_IMAGES = 32;
|
|
3
|
-
export declare
|
|
2
|
+
export declare const CHARS_PER_TOKEN = 4;
|
|
3
|
+
export interface PdfReading {
|
|
4
|
+
readonly ctxSize: number;
|
|
5
|
+
readonly cacheTokens?: number;
|
|
6
|
+
}
|
|
7
|
+
export interface PdfRoom {
|
|
8
|
+
tokens: number;
|
|
9
|
+
images: number;
|
|
10
|
+
}
|
|
11
|
+
export declare function pdfImageRoom({ ctxSize, cacheTokens }: PdfReading, textChars: number): PdfRoom;
|
|
@@ -1,13 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
const PAGE_TOKENS = frameTokens({
|
|
4
|
-
width: MAX_IMAGE_WIDTH,
|
|
5
|
-
height: Math.ceil(MAX_IMAGE_WIDTH * Math.SQRT2)
|
|
6
|
-
});
|
|
7
|
-
const PDF_SHARE = 0.5;
|
|
8
|
-
export const MIN_PDF_PAGE_IMAGES = 2;
|
|
1
|
+
const DOCUMENT_SHARE = 0.5;
|
|
2
|
+
const ANSWER_SHARE = 0.25;
|
|
9
3
|
export const MAX_PDF_PAGE_IMAGES = 32;
|
|
10
|
-
export
|
|
11
|
-
|
|
12
|
-
|
|
4
|
+
export const CHARS_PER_TOKEN = 4;
|
|
5
|
+
export function pdfImageRoom({ ctxSize, cacheTokens }, textChars) {
|
|
6
|
+
const share = ctxSize * DOCUMENT_SHARE;
|
|
7
|
+
const free = cacheTokens === undefined ? share : Math.min(share, ctxSize * (1 - ANSWER_SHARE) - cacheTokens);
|
|
8
|
+
return {
|
|
9
|
+
tokens: Math.max(0, Math.floor(free - textChars / CHARS_PER_TOKEN)),
|
|
10
|
+
images: MAX_PDF_PAGE_IMAGES
|
|
11
|
+
};
|
|
13
12
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { PdfBounds } from '../core/pdf/figure.ts';
|
|
2
|
+
import type { PdfDoc } from '../core/pdf/rasterize.ts';
|
|
3
|
+
import type { PdfRoom } from './pdf-budget.ts';
|
|
4
|
+
export declare const PDF_PAGES_MIME = "application/vnd.qvac.pdf-pages";
|
|
5
|
+
export interface PdfShot {
|
|
6
|
+
readonly page: number;
|
|
7
|
+
readonly clip?: PdfBounds;
|
|
8
|
+
readonly label: string;
|
|
9
|
+
}
|
|
10
|
+
export interface PdfSelection {
|
|
11
|
+
readonly pageCount: number;
|
|
12
|
+
readonly shots: ReadonlyArray<PdfShot>;
|
|
13
|
+
readonly cutAt?: number;
|
|
14
|
+
}
|
|
15
|
+
export declare function pdfSelectionFileName(pdfFileName: string): string;
|
|
16
|
+
export declare function packPdfSelection(selection: PdfSelection): Uint8Array;
|
|
17
|
+
export declare function parsePdfSelection(bytes: Uint8Array): PdfSelection;
|
|
18
|
+
export declare function selectPdfPages(doc: PdfDoc, room: PdfRoom): Promise<PdfSelection>;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import * as z from 'zod';
|
|
3
|
+
import { frameTokens } from './video-budget.js';
|
|
4
|
+
export const PDF_PAGES_MIME = 'application/vnd.qvac.pdf-pages';
|
|
5
|
+
const BoundsSchema = z.object({
|
|
6
|
+
x: z.number(),
|
|
7
|
+
y: z.number(),
|
|
8
|
+
width: z.number().positive(),
|
|
9
|
+
height: z.number().positive()
|
|
10
|
+
});
|
|
11
|
+
const SelectionSchema = z.object({
|
|
12
|
+
version: z.literal(1),
|
|
13
|
+
pageCount: z.number().int().nonnegative(),
|
|
14
|
+
shots: z.array(z.object({
|
|
15
|
+
page: z.number().int().nonnegative(),
|
|
16
|
+
clip: BoundsSchema.optional(),
|
|
17
|
+
label: z.string()
|
|
18
|
+
})),
|
|
19
|
+
cutAt: z.number().int().nonnegative().optional()
|
|
20
|
+
});
|
|
21
|
+
export function pdfSelectionFileName(pdfFileName) {
|
|
22
|
+
return `${pdfFileName}.pages`;
|
|
23
|
+
}
|
|
24
|
+
export function packPdfSelection(selection) {
|
|
25
|
+
return b4a.from(JSON.stringify({ version: 1, ...selection }));
|
|
26
|
+
}
|
|
27
|
+
export function parsePdfSelection(bytes) {
|
|
28
|
+
const { version: _, ...selection } = SelectionSchema.parse(JSON.parse(b4a.toString(bytes)));
|
|
29
|
+
return selection;
|
|
30
|
+
}
|
|
31
|
+
function shotLabel(page, names = []) {
|
|
32
|
+
return names.length ? `page ${page + 1} (${names.join(', ')})` : `page ${page + 1}`;
|
|
33
|
+
}
|
|
34
|
+
export async function selectPdfPages(doc, room) {
|
|
35
|
+
const pageCount = doc.pageCount();
|
|
36
|
+
const shots = [];
|
|
37
|
+
let cutAt;
|
|
38
|
+
for (let page = 0; page < pageCount; page++) {
|
|
39
|
+
let shot;
|
|
40
|
+
try {
|
|
41
|
+
shot = await pictured(doc, page);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
if (!shot) {
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
if (room.images <= 0) {
|
|
50
|
+
return { pageCount, shots, cutAt: cutAt ?? page };
|
|
51
|
+
}
|
|
52
|
+
const cost = frameTokens(doc.renderSize(page, shot.clip));
|
|
53
|
+
if (cost > room.tokens) {
|
|
54
|
+
cutAt ??= page;
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
room.tokens -= cost;
|
|
58
|
+
room.images--;
|
|
59
|
+
shots.push(shot);
|
|
60
|
+
}
|
|
61
|
+
return cutAt === undefined ? { pageCount, shots } : { pageCount, shots, cutAt };
|
|
62
|
+
}
|
|
63
|
+
async function pictured(doc, page) {
|
|
64
|
+
const flags = doc.pageFlags(page);
|
|
65
|
+
if (flags.hasText && !flags.hasImage) {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
if (!flags.hasText && !flags.hasImage && (await doc.blank(page))) {
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
const figure = flags.hasText ? doc.figure(page) : null;
|
|
72
|
+
return figure
|
|
73
|
+
? { page, clip: figure.clip, label: shotLabel(page, figure.names) }
|
|
74
|
+
: { page, label: shotLabel(page) };
|
|
75
|
+
}
|
|
@@ -100,7 +100,7 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
100
100
|
private readonly _lifecycle;
|
|
101
101
|
private readonly _audioTranscripts;
|
|
102
102
|
private readonly _videoProbes;
|
|
103
|
-
private readonly
|
|
103
|
+
private readonly _pdfSelections;
|
|
104
104
|
private readonly _chats;
|
|
105
105
|
private readonly _runImageCacheNames;
|
|
106
106
|
private readonly _reconciledCompactions;
|
|
@@ -229,13 +229,13 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
229
229
|
private _hydrateTurnImages;
|
|
230
230
|
private _hydrateImage;
|
|
231
231
|
private _hydratePdfPages;
|
|
232
|
-
private
|
|
232
|
+
private _pdfReading;
|
|
233
233
|
private _customModel;
|
|
234
|
-
private _hydrateTurnPdfs;
|
|
235
234
|
private _tryHydratePdf;
|
|
236
235
|
private _hydratePdf;
|
|
237
|
-
private
|
|
238
|
-
private
|
|
236
|
+
private _storedPdfSelection;
|
|
237
|
+
private _renderPdfShots;
|
|
238
|
+
private _storePdfSelections;
|
|
239
239
|
private _compactIfNearFull;
|
|
240
240
|
private _runCompaction;
|
|
241
241
|
private _contextReading;
|
|
@@ -21,8 +21,8 @@ import { meshScoped } from './mesh-scoped-engine.js';
|
|
|
21
21
|
import { buildTitlePrompt, deriveChatTitle, isDefaultChatTitle, parseTitleResponse } from './chat-title.js';
|
|
22
22
|
import { matchApprovalRule } from './core/approval-rules.js';
|
|
23
23
|
import { isAudioAttachment } from './core/audio-attachment.js';
|
|
24
|
-
import { attachTurnDocumentIds, formatRagContext, chooseRagNotice, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, createRagSourceTagFilter, citedByPhraseOverlap, formatFullDocumentContext, isFullTextHit, isTruncatedFullTextHit, sourcesFromHits, WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from './core/files/index.js';
|
|
25
|
-
import { annotateDocumentAttachments, documentUnreadNotice, documentUnreadReason, pdfPagesCutNotice,
|
|
24
|
+
import { attachTurnDocumentIds, formatRagContext, chooseRagNotice, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, createRagSourceTagFilter, citedByPhraseOverlap, formatFullDocumentContext, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, sourcesFromHits, WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from './core/files/index.js';
|
|
25
|
+
import { annotateDocumentAttachments, documentUnreadNotice, documentUnreadReason, PDF_PAGES_SOURCE_LINE, pdfPagesCutNotice, pdfPagesShownNotice } from './core/document-attachment.js';
|
|
26
26
|
import { imageCacheKey, imageCacheName, sweepImageCache } from './core/image/cache-sweep.js';
|
|
27
27
|
import { IMAGE_MIMES, normalizeImageForModel } from './core/image/normalize.js';
|
|
28
28
|
import { createPdfRasterizer } from './core/pdf/rasterize.js';
|
|
@@ -51,7 +51,8 @@ import { frameTimes, probeVideo, sampleFrames } from './core/video/frames.js';
|
|
|
51
51
|
import { packVideoDigest, parseVideoDigestHeader, unpackVideoDigest, VIDEO_DIGEST_MIME, videoDigestFileName } from './core/video/digest.js';
|
|
52
52
|
import { VIDEO_FRAME_EDGE, videoFrameCount } from './assistant/video-budget.js';
|
|
53
53
|
import { drainBody, raceInactivity } from './assistant/fetch-body.js';
|
|
54
|
-
import {
|
|
54
|
+
import { pdfImageRoom } from './assistant/pdf-budget.js';
|
|
55
|
+
import { packPdfSelection, parsePdfSelection, PDF_PAGES_MIME, pdfSelectionFileName, selectPdfPages } from './assistant/pdf-pages.js';
|
|
55
56
|
import { effectiveCtxSize } from './api/effective-ctx-size.js';
|
|
56
57
|
import { GENERIC_ACKNOWLEDGEMENTS, HISTORY_TYPES, SKILL_TOOL, TERMINAL_STATES, couldNameSenders, historyFromChunks, inputAvailableToRun, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds } from './assistant/history.js';
|
|
57
58
|
import { errorMessage, isRecord } from './guards.js';
|
|
@@ -114,19 +115,16 @@ const RunOccupancySchema = z.object({
|
|
|
114
115
|
export const CAPTURE_INFERENCE_CALLS_KEY = 'inspector.captureInferenceCalls';
|
|
115
116
|
export const CAPTURE_ON = 'on';
|
|
116
117
|
export const CAPTURE_OFF = 'off';
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
return undefined;
|
|
118
|
+
const NO_PDF_PAGES = { selection: { pageCount: 0, shots: [] }, names: [], labels: [] };
|
|
119
|
+
function pdfNotices(fileName, pages, wholeText) {
|
|
120
|
+
const { cutAt, pageCount } = pages.selection;
|
|
121
|
+
const shown = pages.names.length;
|
|
122
|
+
return [
|
|
123
|
+
...(shown ? [pdfPagesShownNotice(fileName, pages.labels)] : []),
|
|
124
|
+
...(cutAt === undefined
|
|
125
|
+
? []
|
|
126
|
+
: [pdfPagesCutNotice({ fileName, cutAt, pageCount, shown, wholeText })])
|
|
127
|
+
];
|
|
130
128
|
}
|
|
131
129
|
function attachPages(turn, files) {
|
|
132
130
|
if (!files.length)
|
|
@@ -154,8 +152,22 @@ function videoAudioBlock(fileName, { audio, transcript }) {
|
|
|
154
152
|
}
|
|
155
153
|
return unheardAudioNotice(`The audio of the attached video "${fileName}"`, 'transcription failed');
|
|
156
154
|
}
|
|
157
|
-
function
|
|
158
|
-
|
|
155
|
+
async function cachedPdfPages(selection, { dir, pdf, pinned }) {
|
|
156
|
+
const names = selection.shots.map((shot) => pdfShotCacheName(pdf, shot));
|
|
157
|
+
for (const name of names) {
|
|
158
|
+
pinned.add(name);
|
|
159
|
+
}
|
|
160
|
+
const files = names.map((name) => path.join(dir, name));
|
|
161
|
+
const cached = await Promise.all(files.map((file) => cacheOrTouch(file, noBytes)));
|
|
162
|
+
return cached.every(Boolean)
|
|
163
|
+
? { selection, names, labels: selection.shots.map((shot) => shot.label) }
|
|
164
|
+
: null;
|
|
165
|
+
}
|
|
166
|
+
function pdfShotCacheName(pdf, shot) {
|
|
167
|
+
const box = shot.clip
|
|
168
|
+
? `.c${[shot.clip.x, shot.clip.y, shot.clip.width, shot.clip.height].map(Math.round).join('_')}`
|
|
169
|
+
: '';
|
|
170
|
+
return imageCacheName({ blobId: pdf.blobId, fileName: `${pdf.fileName}.p${shot.page}${box}.jpg` });
|
|
159
171
|
}
|
|
160
172
|
async function cacheOrTouch(file, produce) {
|
|
161
173
|
const cached = await fs.promises.stat(file).catch(noop);
|
|
@@ -268,7 +280,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
268
280
|
_lifecycle;
|
|
269
281
|
_audioTranscripts;
|
|
270
282
|
_videoProbes;
|
|
271
|
-
|
|
283
|
+
_pdfSelections;
|
|
272
284
|
_chats;
|
|
273
285
|
_runImageCacheNames;
|
|
274
286
|
_reconciledCompactions;
|
|
@@ -328,7 +340,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
328
340
|
this._lifecycle = new AbortController();
|
|
329
341
|
this._audioTranscripts = new Map();
|
|
330
342
|
this._videoProbes = new Map();
|
|
331
|
-
this.
|
|
343
|
+
this._pdfSelections = new Map();
|
|
332
344
|
this._chats = new Map();
|
|
333
345
|
this._runImageCacheNames = new Map();
|
|
334
346
|
this._reconciledCompactions = new Map();
|
|
@@ -1808,7 +1820,8 @@ export class QvacAssistant extends ReadyResource {
|
|
|
1808
1820
|
const seesImages = sendsFiles && carriesMedia && (await this._seesImages(model));
|
|
1809
1821
|
let unreadableImages = new Map();
|
|
1810
1822
|
const sampled = [];
|
|
1811
|
-
let pdfPages = { rendered: new Set(),
|
|
1823
|
+
let pdfPages = { rendered: new Set(), notices: [] };
|
|
1824
|
+
const selectedPdfs = [];
|
|
1812
1825
|
if (!seesImages) {
|
|
1813
1826
|
const dropped = messages.reduce((count, turn) => count + (turn.images?.length ?? 0), 0);
|
|
1814
1827
|
if (dropped && (this._harness.supportsImages ?? this._info.supportsImages)) {
|
|
@@ -2009,7 +2022,12 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2009
2022
|
this._knowledgeCaps.delete(chatId);
|
|
2010
2023
|
const fitted = knowledge && cap ? { ...knowledge, hits: knowledge.hits.slice(0, cap) } : knowledge;
|
|
2011
2024
|
const injected = fitted
|
|
2012
|
-
? await this._injectKnowledge(fitted, {
|
|
2025
|
+
? await this._injectKnowledge(fitted, {
|
|
2026
|
+
transcript,
|
|
2027
|
+
sourceTagFilter,
|
|
2028
|
+
retrievedRecords,
|
|
2029
|
+
pictured: seesImages && messages.some((turn) => turn.pdfs?.length)
|
|
2030
|
+
})
|
|
2013
2031
|
: { documentsInjected: false, sourcesWritten: false, ragInjected: false };
|
|
2014
2032
|
injectedHits = injected.ragInjected && fitted ? fitted.hits.length : 0;
|
|
2015
2033
|
const hasAudio = messages.some((turn) => turn.audioAttachments?.length || turn.videos?.length);
|
|
@@ -2032,14 +2050,22 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2032
2050
|
}
|
|
2033
2051
|
backfillImageOnlyTurns(messages);
|
|
2034
2052
|
if (seesImages) {
|
|
2053
|
+
const hits = injected.documentsInjected && fitted ? fitted.hits : [];
|
|
2035
2054
|
pdfPages = await this._hydratePdfPages(messages, {
|
|
2036
2055
|
signal: controller.signal,
|
|
2037
2056
|
runId,
|
|
2038
|
-
window: { ctxSize: configuredAgent.ctxSize, modelName: model }
|
|
2057
|
+
window: { ctxSize: configuredAgent.ctxSize, modelName: model },
|
|
2058
|
+
reading: { chatId, meshId, folded },
|
|
2059
|
+
textChars: hits.reduce((chars, hit) => chars + hit.content.length, 0),
|
|
2060
|
+
wholeIds: new Set(hits
|
|
2061
|
+
.filter((hit) => !isTruncatedFullTextHit(hit))
|
|
2062
|
+
.map(fullTextAttachmentId)
|
|
2063
|
+
.filter((id) => id !== null)),
|
|
2064
|
+
selected: selectedPdfs
|
|
2039
2065
|
});
|
|
2040
2066
|
}
|
|
2041
2067
|
annotateDocumentAttachments(messages);
|
|
2042
|
-
for (const { turn, notice } of pdfPages.
|
|
2068
|
+
for (const { turn, notice } of pdfPages.notices) {
|
|
2043
2069
|
turn.content = `${turn.content}\n${notice}`;
|
|
2044
2070
|
}
|
|
2045
2071
|
const unreadIds = attachTurnIds.filter((id) => !pdfPages.rendered.has(id));
|
|
@@ -2247,6 +2273,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2247
2273
|
});
|
|
2248
2274
|
titleOutcome = 'completed';
|
|
2249
2275
|
await this._storeVideoDigests(sampled, { meshId, chatId, runId, deviceId });
|
|
2276
|
+
await this._storePdfSelections(selectedPdfs, { meshId, chatId, runId, deviceId });
|
|
2250
2277
|
}
|
|
2251
2278
|
catch (err) {
|
|
2252
2279
|
const outcome = controller.signal.aborted
|
|
@@ -2885,39 +2912,41 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2885
2912
|
});
|
|
2886
2913
|
return file;
|
|
2887
2914
|
}
|
|
2888
|
-
async _hydratePdfPages(messages, { signal, runId, window }) {
|
|
2915
|
+
async _hydratePdfPages(messages, { signal, runId, window, reading, textChars, wholeIds, selected }) {
|
|
2889
2916
|
const rendered = new Set();
|
|
2890
|
-
const
|
|
2917
|
+
const notices = [];
|
|
2891
2918
|
const turns = messages.filter((turn) => turn.pdfs?.length);
|
|
2892
2919
|
if (!turns.length) {
|
|
2893
|
-
return { rendered,
|
|
2920
|
+
return { rendered, notices };
|
|
2894
2921
|
}
|
|
2895
2922
|
const dir = this._imageCacheDir;
|
|
2896
2923
|
await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
|
|
2897
2924
|
const pinned = this._runImageCacheNames.get(runId) ?? new Set();
|
|
2898
2925
|
this._runImageCacheNames.set(runId, pinned);
|
|
2899
2926
|
const latestUser = messages.findLast((turn) => turn.role === 'user');
|
|
2900
|
-
const
|
|
2927
|
+
const measured = await this._pdfReading(window, reading);
|
|
2901
2928
|
for (const turn of turns) {
|
|
2902
|
-
const
|
|
2903
|
-
|
|
2904
|
-
pinned,
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2929
|
+
const room = pdfImageRoom(measured, turn === latestUser ? textChars : 0);
|
|
2930
|
+
for (const pdf of turn.pdfs ?? []) {
|
|
2931
|
+
const pages = await this._tryHydratePdf({ dir, pdf, pinned, room, signal, selected });
|
|
2932
|
+
attachPages(turn, pages.names.map((name) => path.join(dir, name)));
|
|
2933
|
+
if (pages.names.length && turn === latestUser) {
|
|
2934
|
+
rendered.add(pdf.id);
|
|
2935
|
+
}
|
|
2936
|
+
const whole = turn === latestUser && wholeIds.has(pdf.id);
|
|
2937
|
+
for (const notice of pdfNotices(pdf.fileName, pages, whole)) {
|
|
2938
|
+
notices.push({ turn, notice });
|
|
2911
2939
|
}
|
|
2912
2940
|
}
|
|
2913
|
-
|
|
2941
|
+
delete turn.pdfs;
|
|
2914
2942
|
}
|
|
2915
|
-
return { rendered,
|
|
2943
|
+
return { rendered, notices };
|
|
2916
2944
|
}
|
|
2917
|
-
async
|
|
2945
|
+
async _pdfReading({ ctxSize, modelName }, options) {
|
|
2918
2946
|
const custom = await this._customModel(modelName);
|
|
2919
2947
|
const window = effectiveCtxSize({ ctxSize, modelName }, { appCtxSize: 0, platform: this._platform, custom });
|
|
2920
|
-
|
|
2948
|
+
const { cacheTokens } = await this._contextReading(options, window);
|
|
2949
|
+
return { ctxSize: window, cacheTokens };
|
|
2921
2950
|
}
|
|
2922
2951
|
async _customModel(name) {
|
|
2923
2952
|
for await (const list of this._engine.customModelsWatch({})) {
|
|
@@ -2925,113 +2954,104 @@ export class QvacAssistant extends ReadyResource {
|
|
|
2925
2954
|
}
|
|
2926
2955
|
return null;
|
|
2927
2956
|
}
|
|
2928
|
-
async
|
|
2929
|
-
const shown = [];
|
|
2930
|
-
const notices = [];
|
|
2931
|
-
let budget = maxPages;
|
|
2932
|
-
for (const pdf of turn.pdfs ?? []) {
|
|
2933
|
-
if (budget <= 0) {
|
|
2934
|
-
notices.push(pdfPagesSkippedNotice(pdf.fileName));
|
|
2935
|
-
continue;
|
|
2936
|
-
}
|
|
2937
|
-
const pages = await this._tryHydratePdf(dir, turn, { pdf, pinned, maxPages: budget, signal });
|
|
2938
|
-
budget -= pages.tried;
|
|
2939
|
-
if (pages.names.length) {
|
|
2940
|
-
shown.push(pdf.id);
|
|
2941
|
-
}
|
|
2942
|
-
if (pages.cut) {
|
|
2943
|
-
notices.push(pdfPagesCutNotice({ fileName: pdf.fileName, ...pages.cut }));
|
|
2944
|
-
}
|
|
2945
|
-
}
|
|
2946
|
-
delete turn.pdfs;
|
|
2947
|
-
return { shown, notices };
|
|
2948
|
-
}
|
|
2949
|
-
async _tryHydratePdf(dir, turn, opts) {
|
|
2957
|
+
async _tryHydratePdf(opts) {
|
|
2950
2958
|
try {
|
|
2951
|
-
return await this._hydratePdf(
|
|
2959
|
+
return await this._hydratePdf(opts);
|
|
2952
2960
|
}
|
|
2953
2961
|
catch (err) {
|
|
2954
2962
|
if (opts.signal.aborted) {
|
|
2955
2963
|
throw err;
|
|
2956
2964
|
}
|
|
2957
2965
|
this._log?.warn('assistant', `pdf rasterization failed — ${opts.pdf.fileName}`, err);
|
|
2958
|
-
return
|
|
2966
|
+
return NO_PDF_PAGES;
|
|
2959
2967
|
}
|
|
2960
2968
|
}
|
|
2961
|
-
async _hydratePdf(
|
|
2962
|
-
const
|
|
2963
|
-
const
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2969
|
+
async _hydratePdf(hydration) {
|
|
2970
|
+
const { dir, pdf, pinned, room, signal, selected } = hydration;
|
|
2971
|
+
const key = pdf.pages?.blobId ?? pdf.id;
|
|
2972
|
+
const remember = (selection) => {
|
|
2973
|
+
this._pdfSelections.set(key, selection);
|
|
2974
|
+
if (!pdf.pages) {
|
|
2975
|
+
selected.push({ ref: pdf, selection });
|
|
2967
2976
|
}
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2977
|
+
};
|
|
2978
|
+
const known = this._pdfSelections.get(key) ?? (await this._storedPdfSelection(pdf, signal));
|
|
2979
|
+
if (known) {
|
|
2980
|
+
remember(known);
|
|
2981
|
+
const cached = await cachedPdfPages(known, { dir, pdf, pinned });
|
|
2982
|
+
if (cached) {
|
|
2983
|
+
return cached;
|
|
2972
2984
|
}
|
|
2973
2985
|
}
|
|
2974
|
-
const pages = await this._renderPdf(dir, { pdf, pinned, maxPages, signal });
|
|
2975
|
-
if (!pages) {
|
|
2976
|
-
return { names: [], tried: 0 };
|
|
2977
|
-
}
|
|
2978
|
-
this._pdfPages.set(key, pages);
|
|
2979
|
-
attachPages(turn, pages.names.map((name) => path.join(dir, name)));
|
|
2980
|
-
return pages;
|
|
2981
|
-
}
|
|
2982
|
-
async _renderPdf(dir, { pdf, pinned, maxPages, signal }) {
|
|
2983
2986
|
const bytes = await this._readBlob(pdf.url, signal);
|
|
2984
2987
|
if (signal.aborted) {
|
|
2985
2988
|
throw new Error('hydration aborted');
|
|
2986
2989
|
}
|
|
2987
|
-
|
|
2988
|
-
return null;
|
|
2989
|
-
}
|
|
2990
|
-
const doc = await this._pdfRasterizer.open(bytes);
|
|
2990
|
+
const doc = bytes ? await this._pdfRasterizer.open(bytes) : null;
|
|
2991
2991
|
if (!doc) {
|
|
2992
|
-
return
|
|
2992
|
+
return NO_PDF_PAGES;
|
|
2993
2993
|
}
|
|
2994
2994
|
try {
|
|
2995
|
-
const
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
const pageCount = doc.pageCount();
|
|
2999
|
-
for (; page < pageCount && tried < maxPages; page++) {
|
|
3000
|
-
if (signal.aborted)
|
|
3001
|
-
throw new Error('hydration aborted');
|
|
3002
|
-
const outcome = await this._renderPdfPage(doc, { dir, pdf, page, pinned, signal });
|
|
3003
|
-
if (outcome.tried) {
|
|
3004
|
-
tried++;
|
|
3005
|
-
}
|
|
3006
|
-
if (outcome.name) {
|
|
3007
|
-
names.push(outcome.name);
|
|
3008
|
-
}
|
|
2995
|
+
const selection = known ?? (await selectPdfPages(doc, room));
|
|
2996
|
+
if (!known) {
|
|
2997
|
+
remember(selection);
|
|
3009
2998
|
}
|
|
3010
|
-
return
|
|
2999
|
+
return await this._renderPdfShots(doc, selection, hydration);
|
|
3011
3000
|
}
|
|
3012
3001
|
finally {
|
|
3013
3002
|
doc.close();
|
|
3014
3003
|
}
|
|
3015
3004
|
}
|
|
3016
|
-
async
|
|
3017
|
-
|
|
3005
|
+
async _storedPdfSelection(pdf, signal) {
|
|
3006
|
+
const bytes = pdf.pages ? await this._readBlob(pdf.pages.url, signal) : null;
|
|
3007
|
+
if (!bytes) {
|
|
3008
|
+
return null;
|
|
3009
|
+
}
|
|
3018
3010
|
try {
|
|
3019
|
-
|
|
3020
|
-
if (flags.hasText && !flags.hasImage) {
|
|
3021
|
-
return { tried };
|
|
3022
|
-
}
|
|
3023
|
-
tried = true;
|
|
3024
|
-
const name = pdfPageCacheName(pdf, page);
|
|
3025
|
-
pinned.add(name);
|
|
3026
|
-
const cached = await cacheOrTouch(path.join(dir, name), () => doc.renderPage(page));
|
|
3027
|
-
return { tried, name: cached ? name : undefined };
|
|
3011
|
+
return parsePdfSelection(bytes);
|
|
3028
3012
|
}
|
|
3029
3013
|
catch (err) {
|
|
3030
|
-
|
|
3031
|
-
|
|
3014
|
+
this._log?.warn('assistant', `pdf page choice unreadable — ${pdf.fileName}`, err);
|
|
3015
|
+
return null;
|
|
3016
|
+
}
|
|
3017
|
+
}
|
|
3018
|
+
async _renderPdfShots(doc, selection, { dir, pdf, pinned, signal }) {
|
|
3019
|
+
const names = [];
|
|
3020
|
+
const labels = [];
|
|
3021
|
+
for (const shot of selection.shots) {
|
|
3022
|
+
if (signal.aborted)
|
|
3023
|
+
throw new Error('hydration aborted');
|
|
3024
|
+
const name = pdfShotCacheName(pdf, shot);
|
|
3025
|
+
pinned.add(name);
|
|
3026
|
+
try {
|
|
3027
|
+
const file = path.join(dir, name);
|
|
3028
|
+
if (await cacheOrTouch(file, () => doc.renderPage(shot.page, shot.clip))) {
|
|
3029
|
+
names.push(name);
|
|
3030
|
+
labels.push(shot.label);
|
|
3031
|
+
}
|
|
3032
|
+
}
|
|
3033
|
+
catch (err) {
|
|
3034
|
+
if (signal.aborted) {
|
|
3035
|
+
throw err;
|
|
3036
|
+
}
|
|
3037
|
+
this._log?.warn('assistant', `pdf page skipped — ${pdf.fileName} p${shot.page}`, err);
|
|
3038
|
+
}
|
|
3039
|
+
}
|
|
3040
|
+
return { selection, names, labels };
|
|
3041
|
+
}
|
|
3042
|
+
async _storePdfSelections(selected, target) {
|
|
3043
|
+
for (const { ref, selection } of selected) {
|
|
3044
|
+
try {
|
|
3045
|
+
await this._uploadGenerated({
|
|
3046
|
+
fileName: pdfSelectionFileName(ref.fileName),
|
|
3047
|
+
mimeType: PDF_PAGES_MIME,
|
|
3048
|
+
bytes: packPdfSelection(selection),
|
|
3049
|
+
raw: true
|
|
3050
|
+
}, { ...target, sourceId: ref.id });
|
|
3051
|
+
}
|
|
3052
|
+
catch (err) {
|
|
3053
|
+
this._log?.warn('assistant', `pdf page choice not stored — ${ref.fileName}`, err);
|
|
3032
3054
|
}
|
|
3033
|
-
this._log?.warn('assistant', `pdf page skipped — ${pdf.fileName} p${page}`, err);
|
|
3034
|
-
return { tried };
|
|
3035
3055
|
}
|
|
3036
3056
|
}
|
|
3037
3057
|
async _compactIfNearFull(params) {
|
|
@@ -3706,7 +3726,7 @@ export class QvacAssistant extends ReadyResource {
|
|
|
3706
3726
|
};
|
|
3707
3727
|
}
|
|
3708
3728
|
}
|
|
3709
|
-
async function injectRetrievedKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords, injected }) {
|
|
3729
|
+
async function injectRetrievedKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords, pictured, injected }) {
|
|
3710
3730
|
const records = sourcesFromHits(knowledge.hits, RAG_SOURCE_BOUNDS);
|
|
3711
3731
|
const notice = chooseRagNotice({
|
|
3712
3732
|
errorMessage: knowledge.search.errorMessage,
|
|
@@ -3729,7 +3749,9 @@ async function injectRetrievedKnowledge(knowledge, { transcript, sourceTagFilter
|
|
|
3729
3749
|
});
|
|
3730
3750
|
injected.sourcesWritten = true;
|
|
3731
3751
|
if (records.length > 0) {
|
|
3732
|
-
|
|
3752
|
+
const framing = formatRagContext(knowledge.hits, knowledge.search.availableFiles ?? []);
|
|
3753
|
+
const pages = pictured ? `\n${PDF_PAGES_SOURCE_LINE}` : '';
|
|
3754
|
+
knowledge.lastUser.content = `${framing}${pages}\n\nThe user's message:\n${knowledge.lastUser.content}\n\n${RAG_CITATION_LINE}`;
|
|
3733
3755
|
injected.ragInjected = true;
|
|
3734
3756
|
injected.documentsInjected = true;
|
|
3735
3757
|
}
|
|
@@ -25,11 +25,14 @@ export declare function documentUnreadReason({ knowledgeOn, failedFiles, attache
|
|
|
25
25
|
export declare function documentUnreadNotice(reason: DocumentUnreadReason, plural: boolean): string;
|
|
26
26
|
export interface PdfPagesCutState {
|
|
27
27
|
readonly fileName: string;
|
|
28
|
-
readonly
|
|
28
|
+
readonly cutAt: number;
|
|
29
29
|
readonly pageCount: number;
|
|
30
|
+
readonly shown: number;
|
|
31
|
+
readonly wholeText: boolean;
|
|
30
32
|
}
|
|
31
|
-
export declare function pdfPagesCutNotice({ fileName,
|
|
32
|
-
export declare
|
|
33
|
+
export declare function pdfPagesCutNotice({ fileName, cutAt, pageCount, shown, wholeText }: PdfPagesCutState): string;
|
|
34
|
+
export declare const PDF_PAGES_SOURCE_LINE = "Page images from the PDFs in this chat are a source too: when an image shows what the excerpts do not, answer from the image.";
|
|
35
|
+
export declare function pdfPagesShownNotice(fileName: string, labels: ReadonlyArray<string>): string;
|
|
33
36
|
export interface AnnotatedMessage {
|
|
34
37
|
content: string;
|
|
35
38
|
documents?: DocumentAttachment[];
|
|
@@ -33,11 +33,16 @@ export function documentUnreadNotice(reason, plural) {
|
|
|
33
33
|
}[reason];
|
|
34
34
|
return `[The attached document${plural ? 's' : ''} could not be read for this reply: ${why}.]`;
|
|
35
35
|
}
|
|
36
|
-
export function pdfPagesCutNotice({ fileName,
|
|
37
|
-
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
|
|
36
|
+
export function pdfPagesCutNotice({ fileName, cutAt, pageCount, shown, wholeText }) {
|
|
37
|
+
const missing = shown
|
|
38
|
+
? `Some pictures in "${fileName}" (${pageCount} pages), the first on page ${cutAt + 1}, were not shown to you as images`
|
|
39
|
+
: `No picture in "${fileName}" (${pageCount} pages) was shown to you as an image`;
|
|
40
|
+
const text = wholeText ? '; its text is all above' : '';
|
|
41
|
+
return `[${missing}: they did not fit in the context window, and no tool can show them${text}. If asked about a picture you cannot see, say it could not be shown. Do not guess its content.]`;
|
|
42
|
+
}
|
|
43
|
+
export const PDF_PAGES_SOURCE_LINE = 'Page images from the PDFs in this chat are a source too: when an image shows what the excerpts do not, answer from the image.';
|
|
44
|
+
export function pdfPagesShownNotice(fileName, labels) {
|
|
45
|
+
return `[The images from "${fileName}" are, in order: ${labels.join('; ')}.]`;
|
|
41
46
|
}
|
|
42
47
|
export function annotateDocumentAttachments(messages) {
|
|
43
48
|
for (const turn of messages.filter((message) => message.documents?.length)) {
|
|
@@ -122,7 +122,7 @@ export declare class Files extends EventEmitter {
|
|
|
122
122
|
private _processorStatus;
|
|
123
123
|
close(): Promise<void>;
|
|
124
124
|
}
|
|
125
|
-
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.ts';
|
|
125
|
+
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.ts';
|
|
126
126
|
export type { IndexingEmbedder, RagExtractors, RagSource, VectorIndex } from './rag/index.ts';
|
|
127
127
|
export { attachTurnDocumentIds, groupAttachmentsByMessage } from './records/index.ts';
|
|
128
128
|
export { CoreBlobServer, PENDING_BLOB_REFERENCE, isPendingBlobReference, parseBlobReferenceCoreKey } from './store/index.ts';
|
|
@@ -658,6 +658,6 @@ function fileState(processor, opts) {
|
|
|
658
658
|
}
|
|
659
659
|
return { state: 'pending', reason: 'source unavailable on this device' };
|
|
660
660
|
}
|
|
661
|
-
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.js';
|
|
661
|
+
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.js';
|
|
662
662
|
export { attachTurnDocumentIds, groupAttachmentsByMessage } from './records/index.js';
|
|
663
663
|
export { CoreBlobServer, PENDING_BLOB_REFERENCE, isPendingBlobReference, parseBlobReferenceCoreKey } from './store/index.js';
|
|
@@ -31,6 +31,6 @@ export declare class RagProcessor implements FileProcessor {
|
|
|
31
31
|
export { MAX_KNOWLEDGE_BYTES } from './limits.ts';
|
|
32
32
|
export { canIndexChat } from './policy.ts';
|
|
33
33
|
export type { IndexingEmbedder, IndexingProgress } from './produce.ts';
|
|
34
|
-
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.ts';
|
|
34
|
+
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.ts';
|
|
35
35
|
export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, retrieveRagKnowledge, searchRagChunks, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
|
|
36
36
|
export type { RagArtifactPayload, RagDocumentRef, RagExtractors, RagSource, VectorHit, VectorIndex } from 'rag-extract';
|
|
@@ -70,5 +70,5 @@ export class RagProcessor {
|
|
|
70
70
|
}
|
|
71
71
|
export { MAX_KNOWLEDGE_BYTES } from './limits.js';
|
|
72
72
|
export { canIndexChat } from './policy.js';
|
|
73
|
-
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.js';
|
|
73
|
+
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.js';
|
|
74
74
|
export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, retrieveRagKnowledge, searchRagChunks, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
|
|
@@ -7,6 +7,7 @@ export interface IdentifiedHit {
|
|
|
7
7
|
readonly id: string;
|
|
8
8
|
}
|
|
9
9
|
export declare function isFullTextHit(hit: IdentifiedHit): boolean;
|
|
10
|
+
export declare function fullTextAttachmentId(hit: IdentifiedHit): string | null;
|
|
10
11
|
export type FullTextCoverage = {
|
|
11
12
|
readonly unit: 'pages';
|
|
12
13
|
readonly included: number;
|
|
@@ -21,6 +21,9 @@ const FULL_TEXT_HIT_PREFIX = 'full-text-';
|
|
|
21
21
|
export function isFullTextHit(hit) {
|
|
22
22
|
return hit.id.startsWith(FULL_TEXT_HIT_PREFIX);
|
|
23
23
|
}
|
|
24
|
+
export function fullTextAttachmentId(hit) {
|
|
25
|
+
return isFullTextHit(hit) ? hit.id.slice(FULL_TEXT_HIT_PREFIX.length) : null;
|
|
26
|
+
}
|
|
24
27
|
const FullTextExtrasSchema = z.object({
|
|
25
28
|
truncated: z.boolean().optional().catch(undefined),
|
|
26
29
|
coverage: z
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export interface PdfBounds {
|
|
2
|
+
readonly x: number;
|
|
3
|
+
readonly y: number;
|
|
4
|
+
readonly width: number;
|
|
5
|
+
readonly height: number;
|
|
6
|
+
}
|
|
7
|
+
export interface PdfTextSpan extends PdfBounds {
|
|
8
|
+
readonly text: string;
|
|
9
|
+
}
|
|
10
|
+
export interface PdfFigure {
|
|
11
|
+
readonly clip: PdfBounds;
|
|
12
|
+
readonly names: ReadonlyArray<string>;
|
|
13
|
+
}
|
|
14
|
+
export interface PdfPageLayout {
|
|
15
|
+
readonly size: {
|
|
16
|
+
readonly width: number;
|
|
17
|
+
readonly height: number;
|
|
18
|
+
};
|
|
19
|
+
readonly images: ReadonlyArray<PdfBounds>;
|
|
20
|
+
readonly spans: ReadonlyArray<PdfTextSpan>;
|
|
21
|
+
}
|
|
22
|
+
export declare function figureRegion({ size, images, spans }: PdfPageLayout): PdfFigure | null;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
const CAPTION = /^(?:Figure|Fig\.|Table|Chart|Exhibit)\s+[\w.-]*\w/i;
|
|
2
|
+
const CAPTION_GAP = 72;
|
|
3
|
+
const DECORATION_SHARE = 0.005;
|
|
4
|
+
const WHOLE_PAGE_SHARE = 0.6;
|
|
5
|
+
const MARGIN = 4;
|
|
6
|
+
function boxOf({ x, y, width, height }) {
|
|
7
|
+
return { left: x, bottom: y, right: x + width, top: y + height };
|
|
8
|
+
}
|
|
9
|
+
function join(a, b) {
|
|
10
|
+
return {
|
|
11
|
+
left: Math.min(a.left, b.left),
|
|
12
|
+
bottom: Math.min(a.bottom, b.bottom),
|
|
13
|
+
right: Math.max(a.right, b.right),
|
|
14
|
+
top: Math.max(a.top, b.top)
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
function overlapsAcross(span, box) {
|
|
18
|
+
return span.left < box.right && span.right > box.left;
|
|
19
|
+
}
|
|
20
|
+
function captionBlock(caption, lines, figure) {
|
|
21
|
+
const height = caption.top - caption.bottom;
|
|
22
|
+
let block = caption;
|
|
23
|
+
const below = lines
|
|
24
|
+
.filter((line) => line.top <= caption.top + 1 && line.bottom < caption.bottom)
|
|
25
|
+
.filter((line) => overlapsAcross(line, figure))
|
|
26
|
+
.sort((a, b) => b.top - a.top);
|
|
27
|
+
for (const line of below) {
|
|
28
|
+
if (block.bottom - line.top > height)
|
|
29
|
+
break;
|
|
30
|
+
block = join(block, line);
|
|
31
|
+
}
|
|
32
|
+
return block;
|
|
33
|
+
}
|
|
34
|
+
function captionName(text, line, figure) {
|
|
35
|
+
const name = text.trim().match(CAPTION)?.[0];
|
|
36
|
+
const near = line.top >= figure.bottom - CAPTION_GAP && line.bottom <= figure.top + CAPTION_GAP;
|
|
37
|
+
return name && near && overlapsAcross(line, figure) ? name : null;
|
|
38
|
+
}
|
|
39
|
+
export function figureRegion({ size, images, spans }) {
|
|
40
|
+
const area = size.width * size.height;
|
|
41
|
+
const content = images.filter((image) => image.width * image.height >= DECORATION_SHARE * area);
|
|
42
|
+
const pictures = (content.length ? content : images).map(boxOf);
|
|
43
|
+
if (!pictures.length) {
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
let box = pictures.reduce(join);
|
|
47
|
+
const figure = box;
|
|
48
|
+
const lines = spans.map(boxOf);
|
|
49
|
+
const names = [];
|
|
50
|
+
for (const [index, span] of spans.entries()) {
|
|
51
|
+
const name = captionName(span.text, lines[index], figure);
|
|
52
|
+
if (name) {
|
|
53
|
+
names.push(name);
|
|
54
|
+
box = join(box, captionBlock(lines[index], lines, figure));
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const clip = {
|
|
58
|
+
x: Math.max(0, box.left - MARGIN),
|
|
59
|
+
y: Math.max(0, box.bottom - MARGIN),
|
|
60
|
+
width: Math.min(size.width, box.right + MARGIN) - Math.max(0, box.left - MARGIN),
|
|
61
|
+
height: Math.min(size.height, box.top + MARGIN) - Math.max(0, box.bottom - MARGIN)
|
|
62
|
+
};
|
|
63
|
+
return clip.width * clip.height >= WHOLE_PAGE_SHARE * area ? null : { clip, names };
|
|
64
|
+
}
|
|
@@ -1,14 +1,23 @@
|
|
|
1
1
|
import type { LoggerInterface } from '@qvac/logging';
|
|
2
|
+
import { type PdfBounds, type PdfFigure } from './figure.ts';
|
|
2
3
|
export declare const PDF_RENDER_SCALE = 2;
|
|
3
4
|
interface PageFlags {
|
|
4
5
|
hasImage: boolean;
|
|
5
6
|
hasText: boolean;
|
|
6
7
|
}
|
|
8
|
+
interface PageSize {
|
|
9
|
+
width: number;
|
|
10
|
+
height: number;
|
|
11
|
+
}
|
|
7
12
|
export declare function renderScale(pageWidth: number): number;
|
|
13
|
+
export declare function renderSize(page: PageSize, clip?: PdfBounds): PageSize;
|
|
8
14
|
export interface PdfDoc {
|
|
9
15
|
pageCount(): number;
|
|
10
16
|
pageFlags(page: number): PageFlags;
|
|
11
|
-
|
|
17
|
+
figure(page: number): PdfFigure | null;
|
|
18
|
+
blank(page: number): Promise<boolean>;
|
|
19
|
+
renderSize(page: number, clip?: PdfBounds): PageSize;
|
|
20
|
+
renderPage(page: number, clip?: PdfBounds): Promise<Uint8Array>;
|
|
12
21
|
close(): void;
|
|
13
22
|
}
|
|
14
23
|
export interface PdfRasterizer {
|
|
@@ -1,8 +1,30 @@
|
|
|
1
1
|
import { MAX_IMAGE_WIDTH, rasterToModelJpeg } from '../image/normalize.js';
|
|
2
|
+
import { figureRegion } from './figure.js';
|
|
2
3
|
export const PDF_RENDER_SCALE = 2;
|
|
4
|
+
const BLANK_SCALE = 0.25;
|
|
5
|
+
const PAPER = 230;
|
|
6
|
+
const INK_SHARE = 0.001;
|
|
3
7
|
export function renderScale(pageWidth) {
|
|
4
8
|
return Math.min(PDF_RENDER_SCALE, MAX_IMAGE_WIDTH / pageWidth);
|
|
5
9
|
}
|
|
10
|
+
export function renderSize(page, clip) {
|
|
11
|
+
const scale = renderScale(page.width);
|
|
12
|
+
const width = Math.max(1, Math.round((clip?.width ?? page.width) * scale));
|
|
13
|
+
const height = Math.max(1, Math.round((clip?.height ?? page.height) * scale));
|
|
14
|
+
const shrink = Math.min(1, MAX_IMAGE_WIDTH / width);
|
|
15
|
+
return { width: Math.round(width * shrink), height: Math.round(height * shrink) };
|
|
16
|
+
}
|
|
17
|
+
function blankRaster({ data }) {
|
|
18
|
+
let samples = 0;
|
|
19
|
+
let ink = 0;
|
|
20
|
+
for (let i = 0; i < data.length; i += 4) {
|
|
21
|
+
samples++;
|
|
22
|
+
if (data[i] < PAPER || data[i + 1] < PAPER || data[i + 2] < PAPER) {
|
|
23
|
+
ink++;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return ink <= samples * INK_SHARE;
|
|
27
|
+
}
|
|
6
28
|
let pdfium;
|
|
7
29
|
async function loadPdfium(log) {
|
|
8
30
|
if (pdfium !== undefined)
|
|
@@ -17,6 +39,55 @@ async function loadPdfium(log) {
|
|
|
17
39
|
}
|
|
18
40
|
return pdfium;
|
|
19
41
|
}
|
|
42
|
+
function pageFigure(doc, page) {
|
|
43
|
+
if (doc.pageRotation(page) !== 0) {
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
const images = doc.imageBounds(page);
|
|
47
|
+
if (!images.length) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
const box = doc.pageBox(page);
|
|
51
|
+
const shown = (at) => ({ ...at, x: at.x - box.x, y: at.y - box.y });
|
|
52
|
+
return figureRegion({
|
|
53
|
+
size: { width: box.width, height: box.height },
|
|
54
|
+
images: images.map(shown),
|
|
55
|
+
spans: doc.textSpans(page).map(shown)
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
function wrapDoc(doc) {
|
|
59
|
+
return {
|
|
60
|
+
pageCount() {
|
|
61
|
+
return doc.pageCount();
|
|
62
|
+
},
|
|
63
|
+
pageFlags(page) {
|
|
64
|
+
return doc.pageFlags(page);
|
|
65
|
+
},
|
|
66
|
+
figure(page) {
|
|
67
|
+
return pageFigure(doc, page);
|
|
68
|
+
},
|
|
69
|
+
async blank(page) {
|
|
70
|
+
return blankRaster(await doc.renderAsync(page, { scale: BLANK_SCALE }));
|
|
71
|
+
},
|
|
72
|
+
renderSize(page, clip) {
|
|
73
|
+
return renderSize(doc.pageSize(page), clip);
|
|
74
|
+
},
|
|
75
|
+
async renderPage(page, clip) {
|
|
76
|
+
const scale = renderScale(doc.pageSize(page).width);
|
|
77
|
+
const jpeg = await rasterToModelJpeg(await doc.renderAsync(page, { scale, clip }));
|
|
78
|
+
if (!jpeg) {
|
|
79
|
+
throw new Error('no JPEG encoder: bare-media did not load');
|
|
80
|
+
}
|
|
81
|
+
return jpeg;
|
|
82
|
+
},
|
|
83
|
+
close() {
|
|
84
|
+
try {
|
|
85
|
+
doc.close();
|
|
86
|
+
}
|
|
87
|
+
catch { }
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
}
|
|
20
91
|
export const createPdfRasterizer = (log) => ({
|
|
21
92
|
async open(bytes) {
|
|
22
93
|
const mod = await loadPdfium(log);
|
|
@@ -30,27 +101,6 @@ export const createPdfRasterizer = (log) => ({
|
|
|
30
101
|
catch {
|
|
31
102
|
return null;
|
|
32
103
|
}
|
|
33
|
-
return
|
|
34
|
-
pageCount() {
|
|
35
|
-
return doc.pageCount();
|
|
36
|
-
},
|
|
37
|
-
pageFlags(page) {
|
|
38
|
-
return doc.pageFlags(page);
|
|
39
|
-
},
|
|
40
|
-
async renderPage(page) {
|
|
41
|
-
const scale = renderScale(doc.pageSize(page).width);
|
|
42
|
-
const jpeg = await rasterToModelJpeg(await doc.renderAsync(page, { scale }));
|
|
43
|
-
if (!jpeg) {
|
|
44
|
-
throw new Error('no JPEG encoder: bare-media did not load');
|
|
45
|
-
}
|
|
46
|
-
return jpeg;
|
|
47
|
-
},
|
|
48
|
-
close() {
|
|
49
|
-
try {
|
|
50
|
-
doc.close();
|
|
51
|
-
}
|
|
52
|
-
catch { }
|
|
53
|
-
}
|
|
54
|
-
};
|
|
104
|
+
return wrapDoc(doc);
|
|
55
105
|
}
|
|
56
106
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/core",
|
|
3
|
-
"version": "0.11.
|
|
3
|
+
"version": "0.11.3",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Peer-to-peer AI across your devices",
|
|
6
6
|
"engines": {
|
|
@@ -266,7 +266,7 @@
|
|
|
266
266
|
"bare-module": "~7.4.0",
|
|
267
267
|
"bare-os": "^3.9.1",
|
|
268
268
|
"bare-path": "^3.0.1",
|
|
269
|
-
"bare-pdfium": "^1.
|
|
269
|
+
"bare-pdfium": "^1.8.0",
|
|
270
270
|
"bare-pipe": "^4.2.2",
|
|
271
271
|
"bare-process": "^4.5.1",
|
|
272
272
|
"bare-rpc": "^1.3.7",
|