bunnyquery 1.14.0 → 1.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/bunnyquery.js +143 -17
- package/dist/engine.cjs +139 -15
- package/dist/engine.cjs.map +1 -1
- package/dist/engine.d.mts +181 -29
- package/dist/engine.d.ts +181 -29
- package/dist/engine.mjs +130 -16
- package/dist/engine.mjs.map +1 -1
- package/package.json +1 -1
- package/src/engine/budget.ts +6 -2
- package/src/engine/config.ts +76 -23
- package/src/engine/duplicates.ts +115 -0
- package/src/engine/host.ts +12 -0
- package/src/engine/index.ts +20 -0
- package/src/engine/project_settings.ts +47 -6
- package/src/engine/prompts/index.ts +1 -0
- package/src/engine/prompts/indexing_user_message.ts +23 -0
- package/src/engine/requests.ts +16 -10
- package/src/engine/session.ts +45 -3
package/package.json
CHANGED
package/src/engine/budget.ts
CHANGED
|
@@ -32,6 +32,9 @@ export var CONTEXT_WINDOW_BY_MODEL: Record<string, number> = {
|
|
|
32
32
|
'claude-sonnet-4-5': 1000000, 'claude-sonnet-4': 200000,
|
|
33
33
|
'claude-haiku-4-5': 200000, 'claude-3-5-sonnet': 200000,
|
|
34
34
|
// openai, exact ids
|
|
35
|
+
// gpt-6.1: OpenAI's listing carries no window; taken as the gpt-6 family's until a
|
|
36
|
+
// published figure says otherwise (2026-10-12, the default model).
|
|
37
|
+
'gpt-6.1-sol': 1050000,
|
|
35
38
|
'gpt-6-astra': 1050000, 'gpt-6-sol': 1050000, 'gpt-6-luna': 1050000,
|
|
36
39
|
'gpt-5.6-sol': 1050000, 'gpt-5.6-terra': 1050000, 'gpt-5.6-luna': 1050000,
|
|
37
40
|
'gpt-5.5': 1000000, 'gpt-5.4': 1050000,
|
|
@@ -39,7 +42,7 @@ export var CONTEXT_WINDOW_BY_MODEL: Record<string, number> = {
|
|
|
39
42
|
'gpt-4.1': 1040000, 'gpt-4o': 128000, 'o1': 200000, 'o1-pro': 200000,
|
|
40
43
|
// family keys
|
|
41
44
|
'claude-fable': 1000000, 'claude-opus': 1000000, 'claude-sonnet': 1000000,
|
|
42
|
-
'claude-haiku': 200000, 'gpt-6': 1050000, 'gpt-5.6': 1050000, 'gpt-5': 128000,
|
|
45
|
+
'claude-haiku': 200000, 'gpt-6.1': 1050000, 'gpt-6': 1050000, 'gpt-5.6': 1050000, 'gpt-5': 128000,
|
|
43
46
|
};
|
|
44
47
|
// Two rows above are load-bearing rather than redundant, both because the family
|
|
45
48
|
// walk drops trailing segments:
|
|
@@ -76,6 +79,7 @@ export var MAX_OUTPUT_BY_MODEL: Record<string, number> = {
|
|
|
76
79
|
'claude-sonnet-4-6': 64000, 'claude-haiku-4-5': 64000,
|
|
77
80
|
'claude-3-5-sonnet': 8000,
|
|
78
81
|
// openai
|
|
82
|
+
'gpt-6.1-sol': 128000,
|
|
79
83
|
'gpt-6-astra': 128000, 'gpt-6-sol': 128000, 'gpt-6-luna': 128000,
|
|
80
84
|
'gpt-5.6-sol': 128000, 'gpt-5.6-terra': 128000, 'gpt-5.6-luna': 128000,
|
|
81
85
|
'gpt-5.5': 128000, 'gpt-5.4': 128000,
|
|
@@ -83,7 +87,7 @@ export var MAX_OUTPUT_BY_MODEL: Record<string, number> = {
|
|
|
83
87
|
'gpt-4.1': 16000, 'gpt-4o': 4000, 'o1': 100000, 'o1-pro': 100000,
|
|
84
88
|
// family keys
|
|
85
89
|
'claude-fable': 128000, 'claude-opus': 128000, 'claude-sonnet': 64000,
|
|
86
|
-
'claude-haiku': 64000, 'gpt-6': 128000, 'gpt-5.6': 128000, 'gpt-5': 128000,
|
|
90
|
+
'claude-haiku': 64000, 'gpt-6.1': 128000, 'gpt-6': 128000, 'gpt-5.6': 128000, 'gpt-5': 128000,
|
|
87
91
|
};
|
|
88
92
|
|
|
89
93
|
// The window a project runs at when nobody has touched the setting, clamped per
|
package/src/engine/config.ts
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
*/
|
|
24
24
|
|
|
25
25
|
import { type AttachmentParser, registerAttachmentParser } from './attachment_parsers';
|
|
26
|
-
import { cachedProjectSettings, loadProjectSettings, webAccessFrom } from './project_settings';
|
|
26
|
+
import { cachedProjectSettings, loadProjectSettings, webAccessFrom, skipDuplicateFilesFrom } from './project_settings';
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
29
|
* One report about a turn that is streaming, as handed to `onLiveStreamUpdate`.
|
|
@@ -243,18 +243,29 @@ export interface ChatEngineConfig {
|
|
|
243
243
|
* OpenAI's `web_search` are attached to a chat request only when this
|
|
244
244
|
* resolves to true.
|
|
245
245
|
*
|
|
246
|
-
* OPTIONAL
|
|
247
|
-
*
|
|
248
|
-
*
|
|
249
|
-
*
|
|
250
|
-
*
|
|
251
|
-
*
|
|
252
|
-
*
|
|
246
|
+
* OPTIONAL. The project's own setting (`web_access` in the `bq::settings`
|
|
247
|
+
* record, see project_settings.ts; off unless the owner turned it on) is the
|
|
248
|
+
* CEILING, and this option may only restrict below it:
|
|
249
|
+
* effective = projectSetting && webAccess !== false
|
|
250
|
+
* So `false` here turns the web off for this page whatever the project says,
|
|
251
|
+
* and `true` or unset follows the project. A value above the project setting
|
|
252
|
+
* has no effect: the owner's switch is what lets an answer out onto the web.
|
|
253
253
|
*
|
|
254
254
|
* Chat turns only. An indexing pass keeps the tools it has today, since the
|
|
255
255
|
* worker hands it rendered pages and extracted text and not a URL to fetch.
|
|
256
256
|
*/
|
|
257
257
|
webAccess?: boolean;
|
|
258
|
+
/**
|
|
259
|
+
* Whether a chat attachment that the signed-in user already uploaded to this
|
|
260
|
+
* project (same file name, same size, same uploader) is SKIPPED: not uploaded
|
|
261
|
+
* again, not indexed again, the turn using the stored copy's url. Same
|
|
262
|
+
* ceiling rule as `webAccess`, against the project's `skip_duplicate_files`
|
|
263
|
+
* setting, which is ON unless the owner turned it off:
|
|
264
|
+
* effective = projectSetting && skipDuplicateFiles !== false
|
|
265
|
+
* `false` here lets this page re-upload duplicates on a project that skips
|
|
266
|
+
* them; `true` or unset follows the project. See duplicates.ts for the check.
|
|
267
|
+
*/
|
|
268
|
+
skipDuplicateFiles?: boolean;
|
|
258
269
|
/**
|
|
259
270
|
* skapi.forwardRequestFinalize, bound to the consumer's skapi instance.
|
|
260
271
|
* Stores the version of a streamed turn that history should keep (the engine
|
|
@@ -484,29 +495,71 @@ export function liveStreamingEnabled(): boolean {
|
|
|
484
495
|
}
|
|
485
496
|
|
|
486
497
|
/**
|
|
487
|
-
*
|
|
488
|
-
*
|
|
489
|
-
*
|
|
490
|
-
*
|
|
491
|
-
*
|
|
498
|
+
* THE CEILING RULE, shared by every project switch a host may also set:
|
|
499
|
+
* effective = projectSetting && hostOption !== false
|
|
500
|
+
* The project owner's setting is the most a page can have; a host option can
|
|
501
|
+
* only take it lower. `override` is a per-call host value (a direct caller of
|
|
502
|
+
* callClaudeWithMcp passing `webAccess`), which restricts exactly like the config
|
|
503
|
+
* option: either one being `false` is off.
|
|
504
|
+
*/
|
|
505
|
+
function underCeiling(projectSetting: boolean, option: boolean | undefined, override?: boolean): boolean {
|
|
506
|
+
if (!projectSetting) return false;
|
|
507
|
+
if (option === false) return false;
|
|
508
|
+
if (override === false) return false;
|
|
509
|
+
return true;
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/**
|
|
513
|
+
* The web switch for a chat turn, SYNCHRONOUSLY: the project's cached
|
|
514
|
+
* `web_access` setting (off unless the owner turned it on), restricted by the
|
|
515
|
+
* host's `webAccess` option when that is `false`. For a template or a prompt
|
|
516
|
+
* builder, which have nowhere to await. Reads the cache as it stands, so before
|
|
517
|
+
* the settings fetch settles it answers the default; request builders use
|
|
518
|
+
* resolveWebAccess below instead.
|
|
492
519
|
*/
|
|
493
520
|
export function webAccessEnabled(service?: string): boolean {
|
|
494
521
|
const opt = _config ? _config.webAccess : undefined;
|
|
495
|
-
|
|
496
|
-
return webAccessFrom(service ? cachedProjectSettings(service) : null);
|
|
522
|
+
return underCeiling(webAccessFrom(service ? cachedProjectSettings(service) : null), opt);
|
|
497
523
|
}
|
|
498
524
|
|
|
499
525
|
/**
|
|
500
|
-
* The same switch, awaiting the project's settings record
|
|
501
|
-
*
|
|
502
|
-
*
|
|
503
|
-
*
|
|
526
|
+
* The same switch, awaiting the project's settings record. Never rejects: a
|
|
527
|
+
* failed or impossible read (no reader injected, a signed-out visitor on a
|
|
528
|
+
* require_login project) settles as "unset", which is off. Cheap after the first
|
|
529
|
+
* call per project: the store dedupes and caches. `override` is a per-call host
|
|
530
|
+
* value that may only restrict (see underCeiling).
|
|
504
531
|
*/
|
|
505
|
-
export function resolveWebAccess(service?: string): Promise<boolean> {
|
|
532
|
+
export function resolveWebAccess(service?: string, override?: boolean): Promise<boolean> {
|
|
506
533
|
const opt = _config ? _config.webAccess : undefined;
|
|
507
|
-
if (
|
|
508
|
-
|
|
509
|
-
|
|
534
|
+
if (!service) return Promise.resolve(underCeiling(webAccessFrom(null), opt, override));
|
|
535
|
+
return loadProjectSettings(service)
|
|
536
|
+
.then(webAccessFrom, () => false)
|
|
537
|
+
.then((setting) => underCeiling(setting, opt, override));
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/**
|
|
541
|
+
* The skip-duplicates switch for an upload, SYNCHRONOUSLY: the project's cached
|
|
542
|
+
* `skip_duplicate_files` setting (on unless the owner turned it off), restricted
|
|
543
|
+
* by the host's `skipDuplicateFiles` option when that is `false`. Answers the
|
|
544
|
+
* default before the settings fetch settles; the upload path uses
|
|
545
|
+
* resolveSkipDuplicateFiles below.
|
|
546
|
+
*/
|
|
547
|
+
export function skipDuplicateFilesEnabled(service?: string): boolean {
|
|
548
|
+
const opt = _config ? _config.skipDuplicateFiles : undefined;
|
|
549
|
+
return underCeiling(skipDuplicateFilesFrom(service ? cachedProjectSettings(service) : null), opt);
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
/**
|
|
553
|
+
* The same switch, awaiting the project's settings record. Never rejects: a
|
|
554
|
+
* failed read settles as "unset", which for this switch is ON (the default), so
|
|
555
|
+
* a project that cannot be read still skips duplicates unless the host said no.
|
|
556
|
+
*/
|
|
557
|
+
export function resolveSkipDuplicateFiles(service?: string): Promise<boolean> {
|
|
558
|
+
const opt = _config ? _config.skipDuplicateFiles : undefined;
|
|
559
|
+
if (!service) return Promise.resolve(underCeiling(skipDuplicateFilesFrom(null), opt));
|
|
560
|
+
return loadProjectSettings(service)
|
|
561
|
+
.then(skipDuplicateFilesFrom, () => skipDuplicateFilesFrom(null))
|
|
562
|
+
.then((setting) => underCeiling(setting, opt));
|
|
510
563
|
}
|
|
511
564
|
|
|
512
565
|
/**
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Duplicate-upload detection for chat attachments (reliability plan, owner's
|
|
3
|
+
* yes 2026-10-12): "has the signed-in user already uploaded THIS file to this
|
|
4
|
+
* project?", answered from the file records the upload path already writes.
|
|
5
|
+
*
|
|
6
|
+
* WHAT COUNTS AS THE SAME FILE. Same name, same size in bytes, same uploader.
|
|
7
|
+
* Every upload creates a `src::<storagePath>` record (table `file_summaries`)
|
|
8
|
+
* whose `data.size_bytes` is the byte count and whose `user_id` is the
|
|
9
|
+
* uploader, so a lookup by name narrowed by those two fields finds a file
|
|
10
|
+
* without reading its bytes. A different user's identical file is NOT a
|
|
11
|
+
* duplicate: that user's copy sits at a different access group and path and
|
|
12
|
+
* may be invisible to this one.
|
|
13
|
+
*
|
|
14
|
+
* HOW THE NAME IS FOUND. `getUniqueId` with `condition: '<='` is an ENDS-WITH
|
|
15
|
+
* match on the record id (served from the reversed proxy rows), and a file
|
|
16
|
+
* record's id ends with the file's name: `src::<name>` for a file at the
|
|
17
|
+
* root, `src::<folders>/<name>` otherwise. The match is then narrowed to
|
|
18
|
+
* exactly those two shapes, so `src::report.pdf` is not found by "port.pdf",
|
|
19
|
+
* and to the `src::` prefix, because the `run::` and `done::` index markers
|
|
20
|
+
* end with the same path and are not files.
|
|
21
|
+
*
|
|
22
|
+
* NEVER THROWS. A failed or refused lookup (a signed-out visitor on a
|
|
23
|
+
* require_login project, a region without the proxy rows yet, a network
|
|
24
|
+
* error) answers null, and null means "upload as usual": the check exists to
|
|
25
|
+
* save an indexing run, and must never be able to cost one.
|
|
26
|
+
*
|
|
27
|
+
* TRANSPORT-FREE like the rest of the engine: the caller hands in the skapi
|
|
28
|
+
* instance it owns. The widget passes its own; a host with no instance simply
|
|
29
|
+
* does not wire the hook and uploads as before.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
/** The record found, with the storage path the stored file lives at. */
|
|
33
|
+
export type DuplicateUploadMatch = {
|
|
34
|
+
/** The stored file's db storage path (the record id without `src::`). */
|
|
35
|
+
storagePath: string;
|
|
36
|
+
/** The matching `src::` record id. */
|
|
37
|
+
uniqueId: string;
|
|
38
|
+
/** The file record as read. */
|
|
39
|
+
record: any;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
export type DuplicateUploadFile = {
|
|
43
|
+
/** The file name as it is STORED: the last segment of the storage path. */
|
|
44
|
+
name: string;
|
|
45
|
+
/** Size in bytes. */
|
|
46
|
+
size: number;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/** The `src::` records to read before giving up; a name is rarely shared this widely. */
|
|
50
|
+
export const DUPLICATE_LOOKUP_MAX_CANDIDATES = 25;
|
|
51
|
+
|
|
52
|
+
const SRC_PREFIX = 'src::';
|
|
53
|
+
|
|
54
|
+
/** Narrow an ends-with listing to real file records for exactly this name. */
|
|
55
|
+
export function isDuplicateCandidateId(uniqueId: unknown, name: string): boolean {
|
|
56
|
+
if (typeof uniqueId !== 'string' || !name) return false;
|
|
57
|
+
if (uniqueId.indexOf(SRC_PREFIX) !== 0) return false;
|
|
58
|
+
if (uniqueId === SRC_PREFIX + name) return true;
|
|
59
|
+
const tail = '/' + name;
|
|
60
|
+
return uniqueId.length > SRC_PREFIX.length + tail.length && uniqueId.slice(-tail.length) === tail;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function listOf(res: any): any[] {
|
|
64
|
+
if (!res) return [];
|
|
65
|
+
if (Array.isArray(res)) return res;
|
|
66
|
+
return Array.isArray(res.list) ? res.list : [];
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The first `src::` record for `file.name` whose `data.size_bytes` equals
|
|
71
|
+
* `file.size` and whose `user_id` is `userId`, or null. `opts.service` is passed
|
|
72
|
+
* through to both skapi calls when given (the widget names the project on every
|
|
73
|
+
* call); otherwise the instance's own project is queried.
|
|
74
|
+
*/
|
|
75
|
+
export async function findDuplicateUpload(
|
|
76
|
+
skapi: any,
|
|
77
|
+
file: DuplicateUploadFile,
|
|
78
|
+
userId: string,
|
|
79
|
+
opts?: { service?: string },
|
|
80
|
+
): Promise<DuplicateUploadMatch | null> {
|
|
81
|
+
try {
|
|
82
|
+
if (!skapi || typeof skapi.getUniqueId !== 'function' || typeof skapi.getRecords !== 'function') return null;
|
|
83
|
+
const name = file && typeof file.name === 'string' ? file.name : '';
|
|
84
|
+
const size = file && typeof file.size === 'number' ? file.size : NaN;
|
|
85
|
+
if (!name || !(size >= 0) || !userId) return null;
|
|
86
|
+
const service = opts && opts.service ? { service: opts.service } : {};
|
|
87
|
+
|
|
88
|
+
const found = listOf(await skapi.getUniqueId(Object.assign({ unique_id: name, condition: '<=' }, service)));
|
|
89
|
+
const ids: string[] = [];
|
|
90
|
+
for (const item of found) {
|
|
91
|
+
const id = item && typeof item === 'object' ? item.unique_id : item;
|
|
92
|
+
if (isDuplicateCandidateId(id, name) && ids.indexOf(id) === -1) ids.push(id);
|
|
93
|
+
if (ids.length >= DUPLICATE_LOOKUP_MAX_CANDIDATES) break;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
for (const uniqueId of ids) {
|
|
97
|
+
let record: any = null;
|
|
98
|
+
try {
|
|
99
|
+
record = listOf(await skapi.getRecords(Object.assign({ unique_id: uniqueId }, service)))[0] || null;
|
|
100
|
+
} catch {
|
|
101
|
+
// One unreadable candidate (another group, a deleted record) is not
|
|
102
|
+
// the answer; the next one may be.
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
if (!record || typeof record !== 'object') continue;
|
|
106
|
+
const data = record.data && typeof record.data === 'object' ? record.data : null;
|
|
107
|
+
if (!data || data.size_bytes !== size) continue;
|
|
108
|
+
if (record.user_id !== userId) continue;
|
|
109
|
+
return { storagePath: uniqueId.slice(SRC_PREFIX.length), uniqueId, record };
|
|
110
|
+
}
|
|
111
|
+
return null;
|
|
112
|
+
} catch {
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
}
|
package/src/engine/host.ts
CHANGED
|
@@ -385,6 +385,18 @@ export interface ChatHost {
|
|
|
385
385
|
* setting existed.
|
|
386
386
|
*/
|
|
387
387
|
uploadAccessGroup?(storagePath: string): 'public' | 'authorized' | 'private' | undefined | Promise<'public' | 'authorized' | 'private' | undefined>;
|
|
388
|
+
/**
|
|
389
|
+
* "Has the signed-in user already uploaded this file to this project?" Asked
|
|
390
|
+
* BEFORE a member's bytes go up, only while the skip-duplicates switch resolves
|
|
391
|
+
* on (config.ts resolveSkipDuplicateFiles: the project's `skip_duplicate_files`
|
|
392
|
+
* setting, on by default, restricted by the host's `skipDuplicateFiles: false`).
|
|
393
|
+
* `file.name` is the name as STORED (the last segment of the storage path the
|
|
394
|
+
* host itself produced), `file.size` the byte count. A match means the member
|
|
395
|
+
* is neither uploaded nor indexed: the turn takes the stored copy's url and
|
|
396
|
+
* the chip says so. Null, a rejection or a missing hook all mean "upload as
|
|
397
|
+
* usual"; the engine's findDuplicateUpload is the ready-made implementation.
|
|
398
|
+
*/
|
|
399
|
+
findDuplicateUpload?(file: { name: string; size: number }): Promise<{ storagePath: string } | null> | { storagePath: string } | null;
|
|
388
400
|
/** Map a relative path to the consumer's db storage key (e.g. uid-prefixed). */
|
|
389
401
|
storagePathFor(relPath: string): string;
|
|
390
402
|
getMimeType(name: string): string | null;
|
package/src/engine/index.ts
CHANGED
|
@@ -34,8 +34,23 @@ export {
|
|
|
34
34
|
// template; the awaiting form is what the request builders use.
|
|
35
35
|
webAccessEnabled,
|
|
36
36
|
resolveWebAccess,
|
|
37
|
+
// Same ceiling rule for the upload path's skip-duplicates switch: the project's
|
|
38
|
+
// `skip_duplicate_files` setting (on by default), restricted by a host's
|
|
39
|
+
// `skipDuplicateFiles: false`.
|
|
40
|
+
skipDuplicateFilesEnabled,
|
|
41
|
+
resolveSkipDuplicateFiles,
|
|
37
42
|
} from './config';
|
|
38
43
|
|
|
44
|
+
// "Has this user already uploaded this file here?", answered from the `src::` file
|
|
45
|
+
// records. The widget wires it as the host's findDuplicateUpload hook; it never throws.
|
|
46
|
+
export {
|
|
47
|
+
findDuplicateUpload,
|
|
48
|
+
isDuplicateCandidateId,
|
|
49
|
+
DUPLICATE_LOOKUP_MAX_CANDIDATES,
|
|
50
|
+
type DuplicateUploadMatch,
|
|
51
|
+
type DuplicateUploadFile,
|
|
52
|
+
} from './duplicates';
|
|
53
|
+
|
|
39
54
|
export {
|
|
40
55
|
isServerExtractable,
|
|
41
56
|
isOfficeFile,
|
|
@@ -286,6 +301,11 @@ export {
|
|
|
286
301
|
normalizeWebAccess,
|
|
287
302
|
webAccessFrom,
|
|
288
303
|
projectWebAccess,
|
|
304
|
+
// skip duplicate uploads (same record, its own key, on by default)
|
|
305
|
+
DEFAULT_SKIP_DUPLICATE_FILES,
|
|
306
|
+
normalizeSkipDuplicateFiles,
|
|
307
|
+
skipDuplicateFilesFrom,
|
|
308
|
+
projectSkipDuplicateFiles,
|
|
289
309
|
// pure readers over a settings `data` object
|
|
290
310
|
normalizeUploadAccessGroup,
|
|
291
311
|
normalizeProjectAccessSetting,
|
|
@@ -96,6 +96,7 @@ export type ProjectSettingsData = {
|
|
|
96
96
|
upload_access_group?: unknown;
|
|
97
97
|
chat_greeting?: unknown;
|
|
98
98
|
web_access?: unknown;
|
|
99
|
+
skip_duplicate_files?: unknown;
|
|
99
100
|
[key: string]: unknown;
|
|
100
101
|
};
|
|
101
102
|
|
|
@@ -109,12 +110,14 @@ export type ProjectSettingsData = {
|
|
|
109
110
|
* database alone. Only the literal `true` turns it on: this record is public and
|
|
110
111
|
* a client must never read a value it did not write as permission.
|
|
111
112
|
*
|
|
112
|
-
* Resolution
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
113
|
+
* Resolution, applied by the engine where the chat request is built (config.ts
|
|
114
|
+
* resolveWebAccess): THIS SETTING IS THE CEILING and the host may only restrict.
|
|
115
|
+
* effective = projectSetting && hostOption !== false
|
|
116
|
+
* A host's `webAccess: false` turns a project that is on off for its own page;
|
|
117
|
+
* a host's `true` on a project that is off has no effect, because the owner's
|
|
118
|
+
* switch is the one that lets an answer out onto the web. Indexing requests are
|
|
119
|
+
* not governed by this: an indexing pass keeps the tools it has today (see
|
|
120
|
+
* requests.ts notifyAgentSaveAttachment).
|
|
118
121
|
*/
|
|
119
122
|
export const DEFAULT_WEB_ACCESS = false;
|
|
120
123
|
|
|
@@ -128,6 +131,35 @@ export function webAccessFrom(data: ProjectSettingsData | null | undefined): boo
|
|
|
128
131
|
return data ? normalizeWebAccess(data.web_access) : DEFAULT_WEB_ACCESS;
|
|
129
132
|
}
|
|
130
133
|
|
|
134
|
+
/**
|
|
135
|
+
* Whether an upload that is byte-for-byte the SAME FILE the signed-in user
|
|
136
|
+
* already has in this project (same name, same size, same uploader) is skipped:
|
|
137
|
+
* not uploaded again and not indexed again, with the chat turn using the stored
|
|
138
|
+
* copy. Same record, its own `skip_duplicate_files` key.
|
|
139
|
+
*
|
|
140
|
+
* ON BY DEFAULT: a second copy of a file costs a second indexing run, which is
|
|
141
|
+
* the single most expensive thing a project does, and answers from two copies
|
|
142
|
+
* of one document double every row. Only the literal `false` turns it off: this
|
|
143
|
+
* record is public and a client must never read a value it did not write.
|
|
144
|
+
*
|
|
145
|
+
* Resolution is the same ceiling rule as `web_access` (config.ts
|
|
146
|
+
* resolveSkipDuplicateFiles): effective = projectSetting && hostOption !== false.
|
|
147
|
+
* The host's `skipDuplicateFiles: false` lets its page re-upload duplicates on
|
|
148
|
+
* a project that skips them; a host's `true` on a project that turned it off has
|
|
149
|
+
* no effect.
|
|
150
|
+
*/
|
|
151
|
+
export const DEFAULT_SKIP_DUPLICATE_FILES = true;
|
|
152
|
+
|
|
153
|
+
/** Narrow a stored value to the switch: only the literal `false` is off. */
|
|
154
|
+
export function normalizeSkipDuplicateFiles(value: any): boolean {
|
|
155
|
+
return value !== false;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** The switch held in a settings-record `data`, true when unset. */
|
|
159
|
+
export function skipDuplicateFilesFrom(data: ProjectSettingsData | null | undefined): boolean {
|
|
160
|
+
return data ? normalizeSkipDuplicateFiles(data.skip_duplicate_files) : DEFAULT_SKIP_DUPLICATE_FILES;
|
|
161
|
+
}
|
|
162
|
+
|
|
131
163
|
/**
|
|
132
164
|
* Longest custom opening line the dashboard will store.
|
|
133
165
|
*
|
|
@@ -354,6 +386,15 @@ export function projectWebAccess(service: string): boolean {
|
|
|
354
386
|
return webAccessFrom(cachedProjectSettings(service));
|
|
355
387
|
}
|
|
356
388
|
|
|
389
|
+
/**
|
|
390
|
+
* Sync convenience: the project's skip-duplicates setting, true until the fetch
|
|
391
|
+
* settles or when the project never set it. The upload path prefers the
|
|
392
|
+
* awaiting form (config.ts resolveSkipDuplicateFiles).
|
|
393
|
+
*/
|
|
394
|
+
export function projectSkipDuplicateFiles(service: string): boolean {
|
|
395
|
+
return skipDuplicateFilesFrom(cachedProjectSettings(service));
|
|
396
|
+
}
|
|
397
|
+
|
|
357
398
|
/**
|
|
358
399
|
* Adopt a value the caller just WROTE, so the settings page reflects its own
|
|
359
400
|
* save without a re-fetch.
|
|
@@ -231,6 +231,28 @@ function buildRenderMeta(attachment: IndexingAttachmentInfo): string {
|
|
|
231
231
|
);
|
|
232
232
|
}
|
|
233
233
|
|
|
234
|
+
/**
|
|
235
|
+
* Verbatim page text for SCANNED pages (reliability plan 3.1, owner's yes 2026-10-12).
|
|
236
|
+
*
|
|
237
|
+
* The worker writes a `__PAGES__` record per page. A page WITH a text layer is written
|
|
238
|
+
* server-side before the model call; a page WITHOUT one (a scan) can only come from the
|
|
239
|
+
* model, and the worker's `_fill_vision_pages` fills it from a posted record that names
|
|
240
|
+
* the page (`page`) and carries `page_text`. The signal the model has is the label the
|
|
241
|
+
* worker puts before each image: "Embedded text layer for this page" appears under the
|
|
242
|
+
* label exactly when the page has one (the layer ships the text with the first strip of
|
|
243
|
+
* a page), so "no such block" is "no text layer", and that is the whole rule. Fixed table
|
|
244
|
+
* name, as the system prompt demands for every content kind. Kept short: the owner pays
|
|
245
|
+
* output tokens for the transcription itself, not for the instruction.
|
|
246
|
+
*/
|
|
247
|
+
export const PAGE_TEXT_TABLE = 'page_texts';
|
|
248
|
+
const PAGE_TEXT_RULE =
|
|
249
|
+
`PAGE TEXT for pages WITHOUT a text layer: for EVERY page whose label (its first label, when the page is split ` +
|
|
250
|
+
`into strips) carries NO "Embedded text layer" block, ALSO post exactly ONE record in table "${PAGE_TEXT_TABLE}" ` +
|
|
251
|
+
`with data {"page": <its document page number from the label, 1-based>, "page_text": "<the page's complete text ` +
|
|
252
|
+
`exactly as printed, verbatim, in reading order, nothing summarised or left out>"}, reference "src::" + the storage ` +
|
|
253
|
+
`path above, in addition to the records above. A page WITH an embedded text layer gets NO such record: its text ` +
|
|
254
|
+
`is already stored.\n\n`;
|
|
255
|
+
|
|
234
256
|
// Shared datafy guidance. The placeholder is where the worker splices the note + rendered
|
|
235
257
|
// page images; instructions reference "the page images in this message" so they read
|
|
236
258
|
// correctly whether the images land before or after this text.
|
|
@@ -255,6 +277,7 @@ function buildRenderDatafy(placeholder: string): string {
|
|
|
255
277
|
`summarising. Where a page carries an embedded text layer it is quoted above that page's image: it is the ` +
|
|
256
278
|
`exact text and should be preferred over reading the pixels, with the image used for layout, tables, ` +
|
|
257
279
|
`stamps and handwriting.\n\n` +
|
|
280
|
+
PAGE_TEXT_RULE +
|
|
258
281
|
`Save records for THIS window of pages only, then stop and report what you saved. Do NOT try to read ` +
|
|
259
282
|
`the rest of the file and do NOT worry about the pages after this window: if any remain, the next window ` +
|
|
260
283
|
`is rendered and sent to you automatically. Report only the pages you were actually shown - never imply ` +
|
package/src/engine/requests.ts
CHANGED
|
@@ -56,8 +56,9 @@ const openaiWebSearchTool = () => ({
|
|
|
56
56
|
* nothing when it is off. Off is the default (project_settings.ts
|
|
57
57
|
* DEFAULT_WEB_ACCESS): the answer then comes from the project's files and
|
|
58
58
|
* database alone, which is what the grounding rules promise. The switch is
|
|
59
|
-
* resolved by the caller (resolveWebAccess:
|
|
60
|
-
*
|
|
59
|
+
* resolved by the caller (resolveWebAccess: the project's `web_access` setting
|
|
60
|
+
* is the ceiling, a host option may only restrict) so this stays a pure
|
|
61
|
+
* function of it.
|
|
61
62
|
*/
|
|
62
63
|
const webToolsForChat = (platform: 'claude' | 'openai', webAccess: boolean | undefined): any[] => {
|
|
63
64
|
if (webAccess !== true) return [];
|
|
@@ -66,8 +67,11 @@ const webToolsForChat = (platform: 'claude' | 'openai', webAccess: boolean | und
|
|
|
66
67
|
};
|
|
67
68
|
export const MCP_NAME = 'BunnyQuery';
|
|
68
69
|
|
|
69
|
-
|
|
70
|
-
|
|
70
|
+
// The model a project uses when it never picked one (the settings page shows the same
|
|
71
|
+
// constants). Owner's choice 2026-10-12: Sonnet 5.5 on Claude, GPT-6.1 Sol on OpenAI (both
|
|
72
|
+
// have rows in budget.ts: 1,000,000 / 1,050,000 context, 128,000 output).
|
|
73
|
+
export const DEFAULT_CLAUDE_MODEL = 'claude-sonnet-5-5';
|
|
74
|
+
export const DEFAULT_OPENAI_MODEL = 'gpt-6.1-sol';
|
|
71
75
|
|
|
72
76
|
const mcpUrl = () => chatEngineConfig().mcpBaseUrl;
|
|
73
77
|
|
|
@@ -637,9 +641,11 @@ export type CallClaudeWithMcpParams = {
|
|
|
637
641
|
onResponse?: (res: any) => void;
|
|
638
642
|
onError?: (err: any) => void;
|
|
639
643
|
/**
|
|
640
|
-
* Attach Claude's `web_fetch` to this turn.
|
|
641
|
-
*
|
|
642
|
-
*
|
|
644
|
+
* Attach Claude's `web_fetch` to this turn. A per-call host value under the
|
|
645
|
+
* same ceiling rule as the config option: the project's `web_access` setting
|
|
646
|
+
* is the most a turn can have, and `false` here (or in the config) takes it
|
|
647
|
+
* off. Omitted follows the config option and the project. See config.ts
|
|
648
|
+
* resolveWebAccess.
|
|
643
649
|
*/
|
|
644
650
|
webAccess?: boolean;
|
|
645
651
|
};
|
|
@@ -695,9 +701,9 @@ export async function callClaudeWithMcp({
|
|
|
695
701
|
webAccess,
|
|
696
702
|
}: CallClaudeWithMcpParams) {
|
|
697
703
|
// Resolved HERE and not only in callClaudeWithPublicMcp, so a host calling this
|
|
698
|
-
// builder directly gets the same switch:
|
|
699
|
-
// the
|
|
700
|
-
const web = typeof webAccess === 'boolean' ? webAccess :
|
|
704
|
+
// builder directly gets the same switch: the project's setting is the ceiling,
|
|
705
|
+
// and an explicit `false` (here or in the config) restricts below it.
|
|
706
|
+
const web = await resolveWebAccess(service, typeof webAccess === 'boolean' ? webAccess : undefined);
|
|
701
707
|
const mcpServerDefinition: Record<string, any> = {
|
|
702
708
|
type: 'url',
|
|
703
709
|
name: mcpServer.name,
|
package/src/engine/session.ts
CHANGED
|
@@ -42,7 +42,7 @@ import {
|
|
|
42
42
|
type BgTaskEntry,
|
|
43
43
|
} from './requests';
|
|
44
44
|
import { isPagedReadFile, isImageVisionFile, isWindowedReadFile } from './office';
|
|
45
|
-
import { windowedIndexingEnabled, liveStreamingEnabled, streamRecoveryEnabled, chatEngineConfig, resolveForwardRequestFinalize, resolveForwardRequestStream } from './config';
|
|
45
|
+
import { windowedIndexingEnabled, liveStreamingEnabled, streamRecoveryEnabled, chatEngineConfig, resolveForwardRequestFinalize, resolveForwardRequestStream, resolveSkipDuplicateFiles } from './config';
|
|
46
46
|
// The wire-format knowledge. skapi relays the provider's bytes without reading them,
|
|
47
47
|
// so the session hands them straight to this parser and never inspects a frame itself.
|
|
48
48
|
import { createSseParser, type SseParser } from './sse';
|
|
@@ -5955,11 +5955,35 @@ export class ChatSession {
|
|
|
5955
5955
|
if (!total) return Promise.reject(new Error('Empty attachment'));
|
|
5956
5956
|
var urls: Array<{ name: string; url: string; storagePath: string }> = [];
|
|
5957
5957
|
var anyIndexFailed = false;
|
|
5958
|
+
// Members answered by a file the user already has here (see findDuplicate
|
|
5959
|
+
// below). Counted so a folder chip can say how many it reused.
|
|
5960
|
+
var reusedCount = 0;
|
|
5961
|
+
att.reused = false;
|
|
5958
5962
|
var chain: Promise<any> = Promise.resolve();
|
|
5963
|
+
// "Already uploaded by this user, same name and size" -> the stored copy's
|
|
5964
|
+
// storage path, else null. Asked only while the skip-duplicates switch
|
|
5965
|
+
// resolves ON (the project's setting is the ceiling, the host may only
|
|
5966
|
+
// restrict), and only of a host that wired the hook. Every failure is null,
|
|
5967
|
+
// so this can save an upload and an indexing run but never block one.
|
|
5968
|
+
var findDuplicate = function (member: any): Promise<string | null> {
|
|
5969
|
+
if (typeof self.host.findDuplicateUpload !== 'function') return Promise.resolve(null);
|
|
5970
|
+
var storedName = String(member.storagePath || '').split('/').filter(Boolean).pop() || member.file.name;
|
|
5971
|
+
return resolveSkipDuplicateFiles(id.projectId).then(function (on) {
|
|
5972
|
+
if (!on) return null;
|
|
5973
|
+
return Promise.resolve(self.host.findDuplicateUpload!({ name: storedName, size: member.file.size }))
|
|
5974
|
+
.then(function (hit: any) {
|
|
5975
|
+
return hit && typeof hit.storagePath === 'string' && hit.storagePath ? hit.storagePath : null;
|
|
5976
|
+
});
|
|
5977
|
+
}).catch(function () { return null; });
|
|
5978
|
+
};
|
|
5959
5979
|
members.forEach(function (member: any, idx: number) {
|
|
5960
5980
|
chain = chain.then(function () {
|
|
5961
5981
|
var hadExists = false;
|
|
5962
5982
|
var skipped = false;
|
|
5983
|
+
// The storage path of the file this member DUPLICATES, when the user
|
|
5984
|
+
// already has it here: nothing is uploaded or indexed for it, and the
|
|
5985
|
+
// turn takes the stored copy's url instead.
|
|
5986
|
+
var duplicateOf: string | null = null;
|
|
5963
5987
|
// True when the file already existed and we are re-indexing over it
|
|
5964
5988
|
// (user chose Reindex, OR Overwrite replaced the bytes). Drives the
|
|
5965
5989
|
// delete-then-repost of the stale "src::" record below. Kept separate
|
|
@@ -5978,7 +6002,14 @@ export class ChatSession {
|
|
|
5978
6002
|
onProgress: onProg, setAbort: function (abort) { att._abort = abort; },
|
|
5979
6003
|
});
|
|
5980
6004
|
};
|
|
5981
|
-
return
|
|
6005
|
+
return findDuplicate(member).then(function (dup) {
|
|
6006
|
+
if (dup) {
|
|
6007
|
+
duplicateOf = dup;
|
|
6008
|
+
console.log('[chat-engine] skipping an upload the user already has:', member.storagePath, '->', dup);
|
|
6009
|
+
return;
|
|
6010
|
+
}
|
|
6011
|
+
return doMemberUpload(true);
|
|
6012
|
+
}).catch(function (err: any) {
|
|
5982
6013
|
var code = err && (err.code || (err.body && err.body.code));
|
|
5983
6014
|
var msg = err && (err.message || (err.body && err.body.message) || (typeof err === 'string' ? err : ''));
|
|
5984
6015
|
var isExists = code === 'EXISTS' || (msg && /exist/i.test(msg));
|
|
@@ -5999,9 +6030,20 @@ export class ChatSession {
|
|
|
5999
6030
|
});
|
|
6000
6031
|
}).then(function () {
|
|
6001
6032
|
if (skipped) return; // user skipped this member — no url, no index request
|
|
6002
|
-
|
|
6033
|
+
// A duplicate takes the STORED copy's url: the same reuse an EXISTS
|
|
6034
|
+
// answer gets when the user keeps the file, minus the prompt and
|
|
6035
|
+
// minus the re-index.
|
|
6036
|
+
return self.host.getTemporaryUrl(duplicateOf || member.storagePath);
|
|
6003
6037
|
}).then(function (url: string) {
|
|
6004
6038
|
if (skipped) return; // guard the indexing branch for a skipped member
|
|
6039
|
+
if (duplicateOf) {
|
|
6040
|
+
urls.push({ name: member.relPath, url: url, storagePath: duplicateOf });
|
|
6041
|
+
reusedCount += 1;
|
|
6042
|
+
if (att.kind !== 'folder') { att.uploadedUrl = url; att.storagePath = duplicateOf; att.reused = true; }
|
|
6043
|
+
else att.reusedCount = reusedCount;
|
|
6044
|
+
self.host.renderAttachmentChips();
|
|
6045
|
+
return; // not indexed: the stored copy already is
|
|
6046
|
+
}
|
|
6005
6047
|
urls.push({ name: member.relPath, url: url, storagePath: member.storagePath });
|
|
6006
6048
|
if (att.kind !== 'folder') { att.uploadedUrl = url; att.storagePath = member.storagePath; }
|
|
6007
6049
|
var mime = member.file.type || self.host.getMimeType(member.file.name);
|