@ganju/utils 0.0.8 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channelDebounce.d.ts +11 -0
- package/dist/channelDebounce.d.ts.map +1 -1
- package/dist/channelDebounce.js +12 -1
- package/dist/chunking.d.ts +17 -3
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +125 -19
- package/dist/constants.d.ts +24 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +102 -16
- package/dist/index.d.ts +76 -27
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -0
- package/dist/llmBaseUrl.d.ts +39 -0
- package/dist/llmBaseUrl.d.ts.map +1 -0
- package/dist/llmBaseUrl.js +65 -0
- package/dist/localizeZodIssue.d.ts.map +1 -1
- package/dist/localizeZodIssue.js +6 -0
- package/dist/resourceRerank.d.ts +20 -0
- package/dist/resourceRerank.d.ts.map +1 -0
- package/dist/resourceRerank.js +37 -0
- package/dist/schema.d.ts +41 -24
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +47 -8
- package/dist/syncableResource.d.ts +9 -0
- package/dist/syncableResource.d.ts.map +1 -0
- package/dist/syncableResource.js +31 -0
- package/dist/toolCatalog.d.ts +1 -1
- package/dist/toolCatalog.js +1 -1
- package/package.json +1 -1
- package/src/channelDebounce.ts +36 -0
- package/src/chunking.ts +166 -23
- package/src/constants.ts +114 -16
- package/src/index.ts +26 -1
- package/src/llmBaseUrl.ts +65 -0
- package/src/localizeZodIssue.ts +6 -0
- package/src/resourceRerank.ts +54 -0
- package/src/schema.ts +80 -11
- package/src/syncableResource.ts +48 -0
- package/src/toolCatalog.ts +1 -1
|
@@ -2,7 +2,17 @@ export interface BufferedChannelMessage {
|
|
|
2
2
|
text: string;
|
|
3
3
|
externalMessageId: string | null;
|
|
4
4
|
receivedAt: number;
|
|
5
|
+
immediate?: boolean;
|
|
6
|
+
command?: BufferedChannelCommand | null;
|
|
7
|
+
reply?: Record<string, unknown> | null;
|
|
5
8
|
}
|
|
9
|
+
export interface BufferedChannelCommand {
|
|
10
|
+
promptId: string;
|
|
11
|
+
artifactPromptId: string | null;
|
|
12
|
+
promptTitle: string;
|
|
13
|
+
args: Record<string, string>;
|
|
14
|
+
}
|
|
15
|
+
export declare const batchCommand: (messages: BufferedChannelMessage[]) => BufferedChannelCommand | null;
|
|
6
16
|
export interface ChannelBufferEnvelope {
|
|
7
17
|
channelId: string;
|
|
8
18
|
platform: string;
|
|
@@ -17,6 +27,7 @@ export interface ChannelBufferEnvelope {
|
|
|
17
27
|
export interface ChannelBufferFlush {
|
|
18
28
|
envelope: ChannelBufferEnvelope;
|
|
19
29
|
messages: BufferedChannelMessage[];
|
|
30
|
+
batchId: string;
|
|
20
31
|
}
|
|
21
32
|
export declare const resolveDebounceMs: (config: unknown) => number;
|
|
22
33
|
export declare const channelBufferKey: (channelId: string, externalConversationId: string, externalParticipantId: string) => string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"channelDebounce.d.ts","sourceRoot":"","sources":["../src/channelDebounce.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"channelDebounce.d.ts","sourceRoot":"","sources":["../src/channelDebounce.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IAKnB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,OAAO,CAAC,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAKxC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACxC;AAID,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAID,eAAO,MAAM,YAAY,GACvB,UAAU,sBAAsB,EAAE,KACjC,sBAAsB,GAAG,IAM3B,CAAC;AAOF,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACpD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAGD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,qBAAqB,CAAC;IAChC,QAAQ,EAAE,sBAAsB,EAAE,CAAC;IAGnC,OAAO,EAAE,MAAM,CAAC;CACjB;AAOD,eAAO,MAAM,iBAAiB,GAAI,QAAQ,OAAO,KAAG,MAYnD,CAAC;AAKF,eAAO,MAAM,gBAAgB,GAC3B,WAAW,MAAM,EACjB,wBAAwB,MAAM,EAC9B,uBAAuB,MAAM,KAC5B,MAA2E,CAAC;AAI/E,eAAO,MAAM,oBAAoB,GAC/B,UAAU,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,KAChC,MAIuC,CAAC"}
|
package/dist/channelDebounce.js
CHANGED
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.joinBufferedMessages = exports.channelBufferKey = exports.resolveDebounceMs = void 0;
|
|
3
|
+
exports.joinBufferedMessages = exports.channelBufferKey = exports.resolveDebounceMs = exports.batchCommand = void 0;
|
|
4
4
|
const constants_1 = require("./constants");
|
|
5
|
+
// The prompt a batch runs: the newest command in it, since that is the one the
|
|
6
|
+
// participant asked for last. Null for a batch of plain text.
|
|
7
|
+
const batchCommand = (messages) => {
|
|
8
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
9
|
+
const command = messages[i].command;
|
|
10
|
+
if (command)
|
|
11
|
+
return command;
|
|
12
|
+
}
|
|
13
|
+
return null;
|
|
14
|
+
};
|
|
15
|
+
exports.batchCommand = batchCommand;
|
|
5
16
|
// Read the channel's debounce window off its `config`, clamped to the supported
|
|
6
17
|
// range. An unset value takes the default; an explicit 0 disables buffering, so
|
|
7
18
|
// the channel answers every message as it arrives. Anything malformed (a string,
|
package/dist/chunking.d.ts
CHANGED
|
@@ -6,6 +6,17 @@ export declare const chunkText: (text: string, options?: {
|
|
|
6
6
|
targetChars?: number;
|
|
7
7
|
overlapChars?: number;
|
|
8
8
|
}) => string[];
|
|
9
|
+
export type HeadingSyntax = 'markdown' | 'html';
|
|
10
|
+
/**
|
|
11
|
+
* Which heading syntax a document is written in, or null when it has none we
|
|
12
|
+
* can trust. Plain text is left out on purpose: a line that happens to start
|
|
13
|
+
* with "# " there is as likely a comment or a list as a title.
|
|
14
|
+
*/
|
|
15
|
+
export declare const headingSyntaxFor: (input: {
|
|
16
|
+
mimeType?: string | null;
|
|
17
|
+
fileName?: string | null;
|
|
18
|
+
isWebPage?: boolean;
|
|
19
|
+
}) => HeadingSyntax | null;
|
|
9
20
|
export declare const buildHeader: (input: {
|
|
10
21
|
title?: string | null;
|
|
11
22
|
description?: string | null;
|
|
@@ -13,9 +24,10 @@ export declare const buildHeader: (input: {
|
|
|
13
24
|
mimeType?: string | null;
|
|
14
25
|
fileName?: string | null;
|
|
15
26
|
}) => string;
|
|
16
|
-
export interface ChunkMetadata extends ExtractedDocumentMetadata {
|
|
17
|
-
chunkIndexInPage
|
|
18
|
-
chunksInPage
|
|
27
|
+
export interface ChunkMetadata extends Partial<ExtractedDocumentMetadata> {
|
|
28
|
+
chunkIndexInPage?: number;
|
|
29
|
+
chunksInPage?: number;
|
|
30
|
+
headingPath?: string[];
|
|
19
31
|
}
|
|
20
32
|
export interface PreparedChunk {
|
|
21
33
|
content: string;
|
|
@@ -24,5 +36,7 @@ export interface PreparedChunk {
|
|
|
24
36
|
export declare const prepareChunks: (header: string, documents: ExtractedDocument[] | null, fallbackContent: string | null, options?: {
|
|
25
37
|
targetChars?: number;
|
|
26
38
|
overlapChars?: number;
|
|
39
|
+
mimeType?: string | null;
|
|
40
|
+
fileName?: string | null;
|
|
27
41
|
}) => PreparedChunk[];
|
|
28
42
|
//# sourceMappingURL=chunking.d.ts.map
|
package/dist/chunking.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,KAAK,EACV,iBAAiB,EACjB,yBAAyB,EAC1B,MAAM,qBAAqB,CAAC;AAE7B,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,SAAS,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AASpE,eAAO,MAAM,cAAc,GACzB,MAAM,MAAM,EACZ,QAAQ,MAAM,EACd,aAAY,SAAS,SAAS,EAA+B,KAC5D,MAAM,EA8BR,CAAC;AA4BF,eAAO,MAAM,SAAS,GACpB,MAAM,MAAM,EACZ,UAAU;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KACxD,MAAM,EAAoD,CAAC;AAE9D,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,MAAM,CAAC;AAehD;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,GAAI,OAAO;IACtC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,KAAG,aAAa,GAAG,IAUnB,CAAC;AAuEF,eAAO,MAAM,WAAW,GAAI,OAAO;IACjC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KAAG,MASW,CAAC;AAIhB,MAAM,WAAW,aAAc,SAAQ,OAAO,CAAC,yBAAyB,CAAC;IACvE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CAChC;AAID,eAAO,MAAM,aAAa,GACxB,QAAQ,MAAM,EACd,WAAW,iBAAiB,EAAE,GAAG,IAAI,EACrC,iBAAiB,MAAM,GAAG,IAAI,EAC9B,UAAU;IACR,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KACA,aAAa,EA4Cf,CAAC"}
|
package/dist/chunking.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.prepareChunks = exports.buildHeader = exports.chunkText = exports.splitRecursive = void 0;
|
|
3
|
+
exports.prepareChunks = exports.buildHeader = exports.headingSyntaxFor = exports.chunkText = exports.splitRecursive = void 0;
|
|
4
4
|
const constants_1 = require("./constants");
|
|
5
|
+
const sanitize_1 = require("./sanitize");
|
|
5
6
|
const trySplit = (text, sep) => {
|
|
6
7
|
if (typeof sep === 'string') {
|
|
7
8
|
return text.includes(sep) ? text.split(sep) : null;
|
|
@@ -40,23 +41,115 @@ const splitRecursive = (text, target, separators = constants_1.constants.CHUNK_S
|
|
|
40
41
|
return chunks;
|
|
41
42
|
};
|
|
42
43
|
exports.splitRecursive = splitRecursive;
|
|
43
|
-
|
|
44
|
+
// Each piece keeps its `raw` text — the slice of the input it covers, without
|
|
45
|
+
// the overlap borrowed from the piece before — so a caller can walk the input
|
|
46
|
+
// in order without reading any of it twice.
|
|
47
|
+
const chunkPieces = (text, options) => {
|
|
44
48
|
const target = options?.targetChars ?? constants_1.constants.CHUNK_TARGET_CHARS;
|
|
45
49
|
const overlap = options?.overlapChars ?? constants_1.constants.CHUNK_OVERLAP_CHARS;
|
|
46
50
|
const trimmed = text.trim();
|
|
47
51
|
if (!trimmed)
|
|
48
52
|
return [];
|
|
49
53
|
if (trimmed.length <= target)
|
|
50
|
-
return [trimmed];
|
|
54
|
+
return [{ raw: trimmed, content: trimmed }];
|
|
51
55
|
const rawChunks = (0, exports.splitRecursive)(trimmed, target);
|
|
52
|
-
const
|
|
56
|
+
const pieces = [];
|
|
53
57
|
for (let i = 0; i < rawChunks.length; i++) {
|
|
54
58
|
const prev = i > 0 ? rawChunks[i - 1].slice(-overlap) : '';
|
|
55
|
-
|
|
59
|
+
pieces.push({
|
|
60
|
+
raw: rawChunks[i],
|
|
61
|
+
content: (prev ? prev + '\n' : '') + rawChunks[i]
|
|
62
|
+
});
|
|
56
63
|
}
|
|
57
|
-
return
|
|
64
|
+
return pieces.filter(p => p.content.trim().length > 0);
|
|
58
65
|
};
|
|
66
|
+
const chunkText = (text, options) => chunkPieces(text, options).map(p => p.content);
|
|
59
67
|
exports.chunkText = chunkText;
|
|
68
|
+
const MARKDOWN_HEADING = /^ {0,3}(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/;
|
|
69
|
+
const MARKDOWN_FENCE = /^ {0,3}(```|~~~)/;
|
|
70
|
+
const HTML_HEADING = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi;
|
|
71
|
+
const MAX_HEADING_CHARS = 120;
|
|
72
|
+
const MARKDOWN_FILE_NAME = /\.(md|markdown|mdx)$/i;
|
|
73
|
+
/**
|
|
74
|
+
* Which heading syntax a document is written in, or null when it has none we
|
|
75
|
+
* can trust. Plain text is left out on purpose: a line that happens to start
|
|
76
|
+
* with "# " there is as likely a comment or a list as a title.
|
|
77
|
+
*/
|
|
78
|
+
const headingSyntaxFor = (input) => {
|
|
79
|
+
// Crawled pages are flattened to text with their headings written back as
|
|
80
|
+
// Markdown lines, whatever the page's own content type was.
|
|
81
|
+
if (input.isWebPage)
|
|
82
|
+
return 'markdown';
|
|
83
|
+
if (input.mimeType === constants_1.constants.MIMETYPE_TEXT_MARKDOWN)
|
|
84
|
+
return 'markdown';
|
|
85
|
+
if (input.mimeType === constants_1.constants.MIMETYPE_TEXT_HTML)
|
|
86
|
+
return 'html';
|
|
87
|
+
if (input.fileName && MARKDOWN_FILE_NAME.test(input.fileName)) {
|
|
88
|
+
return 'markdown';
|
|
89
|
+
}
|
|
90
|
+
return null;
|
|
91
|
+
};
|
|
92
|
+
exports.headingSyntaxFor = headingSyntaxFor;
|
|
93
|
+
const cleanHeading = (text) => text.replace(/\s+/g, ' ').trim().slice(0, MAX_HEADING_CHARS);
|
|
94
|
+
const scanMarkdownHeadings = (text, state) => {
|
|
95
|
+
const events = [];
|
|
96
|
+
let offset = 0;
|
|
97
|
+
for (const line of text.split('\n')) {
|
|
98
|
+
// A `#` inside a fenced block is a shell comment or a preprocessor line,
|
|
99
|
+
// not a title — and a fence can open in one chunk and close in the next.
|
|
100
|
+
if (MARKDOWN_FENCE.test(line)) {
|
|
101
|
+
state.inFence = !state.inFence;
|
|
102
|
+
}
|
|
103
|
+
else if (!state.inFence) {
|
|
104
|
+
const match = MARKDOWN_HEADING.exec(line);
|
|
105
|
+
const heading = match ? cleanHeading(match[2]) : '';
|
|
106
|
+
if (match && heading) {
|
|
107
|
+
events.push({ offset, level: match[1].length, text: heading });
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
offset += line.length + 1;
|
|
111
|
+
}
|
|
112
|
+
return events;
|
|
113
|
+
};
|
|
114
|
+
const scanHtmlHeadings = (text) => [...text.matchAll(HTML_HEADING)]
|
|
115
|
+
.map(match => ({
|
|
116
|
+
offset: match.index ?? 0,
|
|
117
|
+
level: Number(match[1]),
|
|
118
|
+
text: cleanHeading((0, sanitize_1.decodeEntities)(match[2].replace(/<[^>]*>/g, ' ')))
|
|
119
|
+
}))
|
|
120
|
+
.filter(event => event.text);
|
|
121
|
+
/**
|
|
122
|
+
* Follows the heading hierarchy across a document's chunks, fed in order.
|
|
123
|
+
* Returns, for each chunk, the path of headings it sits under — the section a
|
|
124
|
+
* reader would be pointed to. A chunk that opens with a heading belongs to that
|
|
125
|
+
* heading; one that starts mid-section belongs to the section it continues,
|
|
126
|
+
* even when a later heading begins inside it.
|
|
127
|
+
*/
|
|
128
|
+
const createHeadingTracker = (syntax) => {
|
|
129
|
+
let stack = [];
|
|
130
|
+
const fence = { inFence: false };
|
|
131
|
+
const apply = (event) => {
|
|
132
|
+
stack = stack.filter(h => h.level < event.level);
|
|
133
|
+
stack.push({ level: event.level, text: event.text });
|
|
134
|
+
};
|
|
135
|
+
return (raw) => {
|
|
136
|
+
const events = syntax === 'markdown'
|
|
137
|
+
? scanMarkdownHeadings(raw, fence)
|
|
138
|
+
: scanHtmlHeadings(raw);
|
|
139
|
+
const [first, ...rest] = events;
|
|
140
|
+
// Before a document's first heading there is no section to continue — only
|
|
141
|
+
// the metadata header, or the markup around an HTML body — so a chunk that
|
|
142
|
+
// holds the first heading belongs to it wherever in the chunk it falls.
|
|
143
|
+
const opensWithHeading = first !== undefined &&
|
|
144
|
+
(stack.length === 0 || raw.slice(0, first.offset).trim() === '');
|
|
145
|
+
if (opensWithHeading)
|
|
146
|
+
apply(first);
|
|
147
|
+
const path = stack.map(h => h.text);
|
|
148
|
+
for (const event of opensWithHeading ? rest : events)
|
|
149
|
+
apply(event);
|
|
150
|
+
return path;
|
|
151
|
+
};
|
|
152
|
+
};
|
|
60
153
|
const buildHeader = (input) => [
|
|
61
154
|
input.title ? `Title: ${input.title}` : null,
|
|
62
155
|
input.description ? `Description: ${input.description}` : null,
|
|
@@ -67,37 +160,50 @@ const buildHeader = (input) => [
|
|
|
67
160
|
.filter(Boolean)
|
|
68
161
|
.join('\n');
|
|
69
162
|
exports.buildHeader = buildHeader;
|
|
163
|
+
const CONTROL_CHARS = /[\x00-\x08\x0B\x0C\x0E-\x1F]/g;
|
|
70
164
|
const prepareChunks = (header, documents, fallbackContent, options) => {
|
|
71
165
|
if (documents && documents.length > 0) {
|
|
72
166
|
const out = [];
|
|
73
167
|
documents.forEach((doc, docIndex) => {
|
|
74
|
-
const body = (doc.pageContent || '')
|
|
75
|
-
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
|
|
76
|
-
.trim();
|
|
168
|
+
const body = (doc.pageContent || '').replace(CONTROL_CHARS, '').trim();
|
|
77
169
|
if (!body)
|
|
78
170
|
return;
|
|
79
171
|
const text = docIndex === 0 && header ? `${header}\n\n${body}` : body;
|
|
80
|
-
const
|
|
172
|
+
const syntax = (0, exports.headingSyntaxFor)({
|
|
173
|
+
mimeType: doc.metadata?.source?.mimeType ?? options?.mimeType,
|
|
174
|
+
fileName: doc.metadata?.source?.fileName ?? options?.fileName,
|
|
175
|
+
isWebPage: !!doc.metadata?.web
|
|
176
|
+
});
|
|
177
|
+
const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
|
|
178
|
+
const pieces = chunkPieces(text, options);
|
|
81
179
|
pieces.forEach((piece, pieceIndex) => {
|
|
180
|
+
const headingPath = headingPathOf?.(piece.raw) ?? [];
|
|
82
181
|
out.push({
|
|
83
|
-
content: piece,
|
|
182
|
+
content: piece.content,
|
|
84
183
|
metadata: {
|
|
85
184
|
...doc.metadata,
|
|
86
185
|
chunkIndexInPage: pieceIndex,
|
|
87
|
-
chunksInPage: pieces.length
|
|
186
|
+
chunksInPage: pieces.length,
|
|
187
|
+
...(headingPath.length > 0 ? { headingPath } : {})
|
|
88
188
|
}
|
|
89
189
|
});
|
|
90
190
|
});
|
|
91
191
|
});
|
|
92
192
|
return out;
|
|
93
193
|
}
|
|
94
|
-
const body = (fallbackContent || '')
|
|
95
|
-
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
|
|
96
|
-
.trim();
|
|
194
|
+
const body = (fallbackContent || '').replace(CONTROL_CHARS, '').trim();
|
|
97
195
|
const fullText = [header, body].filter(Boolean).join('\n\n');
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
})
|
|
196
|
+
const syntax = (0, exports.headingSyntaxFor)({
|
|
197
|
+
mimeType: options?.mimeType,
|
|
198
|
+
fileName: options?.fileName
|
|
199
|
+
});
|
|
200
|
+
const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
|
|
201
|
+
return chunkPieces(fullText, options).map(piece => {
|
|
202
|
+
const headingPath = headingPathOf?.(piece.raw) ?? [];
|
|
203
|
+
return {
|
|
204
|
+
content: piece.content,
|
|
205
|
+
metadata: headingPath.length > 0 ? { headingPath } : null
|
|
206
|
+
};
|
|
207
|
+
});
|
|
102
208
|
};
|
|
103
209
|
exports.prepareChunks = prepareChunks;
|
package/dist/constants.d.ts
CHANGED
|
@@ -6,6 +6,8 @@ declare const ONE_DRIVE_TAB_MY_FILES: "my-files";
|
|
|
6
6
|
declare const ONE_DRIVE_TAB_SHARED_WITH_ME: "shared-with-me";
|
|
7
7
|
declare const ONE_DRIVE_TAB_RECENT: "recent";
|
|
8
8
|
declare const ONE_DRIVE_TAB_DRIVES: "drives";
|
|
9
|
+
declare const RESOURCE_SYNC_INTERVAL_DAILY: "daily";
|
|
10
|
+
declare const RESOURCE_SYNC_INTERVAL_WEEKLY: "weekly";
|
|
9
11
|
declare const LLM_PROVIDER_ANTHROPIC: "anthropic";
|
|
10
12
|
declare const LLM_PROVIDER_OPENAI: "openai";
|
|
11
13
|
declare const LLM_PROVIDER_GOOGLE: "google";
|
|
@@ -59,6 +61,7 @@ interface PlanLimits {
|
|
|
59
61
|
toolCallHardCap: number | null;
|
|
60
62
|
includedMessages: number;
|
|
61
63
|
includedEmbeddedBytes: number;
|
|
64
|
+
autoSyncIntervals: ReadonlyArray<typeof RESOURCE_SYNC_INTERVAL_DAILY | typeof RESOURCE_SYNC_INTERVAL_WEEKLY>;
|
|
62
65
|
}
|
|
63
66
|
export type { PlanLimits };
|
|
64
67
|
export declare const constants: {
|
|
@@ -104,6 +107,7 @@ export declare const constants: {
|
|
|
104
107
|
PLAN_FEATURE_EMBEDDED_STORAGE: "embeddedStorage";
|
|
105
108
|
PLAN_FEATURE_MESSAGE: "message";
|
|
106
109
|
PLAN_FEATURE_TOOL_CALL: "toolCall";
|
|
110
|
+
PLAN_FEATURE_RESOURCE_SYNC: "resourceSync";
|
|
107
111
|
PLAN_LIMIT_ERROR_CODE: string;
|
|
108
112
|
BILLING_METER_MESSAGES: string;
|
|
109
113
|
BILLING_METER_SHARED_MESSAGES: string;
|
|
@@ -309,6 +313,17 @@ export declare const constants: {
|
|
|
309
313
|
RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER: "ONE_DRIVE_FOLDER";
|
|
310
314
|
RESOURCE_SOURCE_TYPE_CUSTOM_CODE: "CUSTOM_CODE";
|
|
311
315
|
RESOURCE_SOURCE_TYPES: ("FILE" | "WEBSITE" | "GOOGLE_DRIVE_FOLDER" | "ONE_DRIVE_FOLDER" | "CUSTOM_CODE")[];
|
|
316
|
+
RESOURCE_SYNC_INTERVAL_OFF: "off";
|
|
317
|
+
RESOURCE_SYNC_INTERVAL_DAILY: "daily";
|
|
318
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY: "weekly";
|
|
319
|
+
RESOURCE_SYNC_INTERVALS: readonly ["off", "daily", "weekly"];
|
|
320
|
+
RESOURCE_SYNC_INTERVAL_DEFAULT: "weekly";
|
|
321
|
+
RESOURCE_SYNC_INTERVAL_MS: {
|
|
322
|
+
readonly daily: number;
|
|
323
|
+
readonly weekly: number;
|
|
324
|
+
};
|
|
325
|
+
RESOURCE_SYNC_BATCH_SIZE: number;
|
|
326
|
+
RESOURCE_SYNC_MANUAL_COOLDOWN_MS: number;
|
|
312
327
|
CRAWL_RENDERER_CHEERIO: "cheerio";
|
|
313
328
|
CRAWL_RENDERER_PLAYWRIGHT: "playwright";
|
|
314
329
|
CRAWL_RENDERERS: ("cheerio" | "playwright")[];
|
|
@@ -362,6 +377,9 @@ export declare const constants: {
|
|
|
362
377
|
model: string;
|
|
363
378
|
label: string;
|
|
364
379
|
}[];
|
|
380
|
+
LLM_CATALOG_CUSTOM_KEY: string;
|
|
381
|
+
LLM_BASE_URL_REQUIRED_MESSAGE: string;
|
|
382
|
+
LLM_BASE_URL_INVALID_MESSAGE: string;
|
|
365
383
|
LLM_SYSTEM_DEFAULT: string;
|
|
366
384
|
MAX_TOOL_LOOPS: number;
|
|
367
385
|
CHANNEL_HISTORY_LIMIT: number;
|
|
@@ -431,6 +449,12 @@ export declare const constants: {
|
|
|
431
449
|
CHUNK_TARGET_CHARS: number;
|
|
432
450
|
CHUNK_OVERLAP_CHARS: number;
|
|
433
451
|
EMBED_BATCH_SIZE: number;
|
|
452
|
+
RESOURCE_SEARCH_CANDIDATES: number;
|
|
453
|
+
RESOURCE_SEARCH_RRF_K: number;
|
|
454
|
+
RESOURCE_SEARCH_COMMON_TERM_DIVISOR: number;
|
|
455
|
+
RESOURCE_SEARCH_COMMON_TERM_MIN: number;
|
|
456
|
+
RESOURCE_SEARCH_RERANK_CANDIDATES: number;
|
|
457
|
+
RESOURCE_SEARCH_RERANK_MODEL: string;
|
|
434
458
|
RESOURCE_HANDLER_SLEEP_AFTER: string;
|
|
435
459
|
DOCS_URL: string;
|
|
436
460
|
BASE64_DATA_URI_RE: RegExp;
|
package/dist/constants.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AA6JA,QAAA,MAAM,yBAAyB,EAAiB,UAAU,CAAC;AAC3D,QAAA,MAAM,+BAA+B,EAAuB,gBAAgB,CAAC;AAC7E,QAAA,MAAM,8BAA8B,EAAsB,eAAe,CAAC;AAC1E,QAAA,MAAM,wBAAwB,EAAgB,SAAS,CAAC;AAuCxD,QAAA,MAAM,sBAAsB,EAAiB,UAAU,CAAC;AACxD,QAAA,MAAM,4BAA4B,EAAuB,gBAAgB,CAAC;AAC1E,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAClD,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AA6JA,QAAA,MAAM,yBAAyB,EAAiB,UAAU,CAAC;AAC3D,QAAA,MAAM,+BAA+B,EAAuB,gBAAgB,CAAC;AAC7E,QAAA,MAAM,8BAA8B,EAAsB,eAAe,CAAC;AAC1E,QAAA,MAAM,wBAAwB,EAAgB,SAAS,CAAC;AAuCxD,QAAA,MAAM,sBAAsB,EAAiB,UAAU,CAAC;AACxD,QAAA,MAAM,4BAA4B,EAAuB,gBAAgB,CAAC;AAC1E,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAClD,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAoFlD,QAAA,MAAM,4BAA4B,EAAc,OAAO,CAAC;AACxD,QAAA,MAAM,6BAA6B,EAAe,QAAQ,CAAC;AAyU3D,QAAA,MAAM,sBAAsB,EAAkB,WAAW,CAAC;AAC1D,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAGjD,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAwVjD,MAAM,MAAM,mBAAmB,GAC3B;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC9D;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA0+BpE,UAAU,UAAU;IAElB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAGhC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAMjC,sBAAsB,EAAE,MAAM,CAAC;IAM/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,SAAS,EAAE,OAAO,CAAC;IAKnB,eAAe,EAAE,OAAO,CAAC;IAOzB,gBAAgB,EAAE,OAAO,CAAC;IAK1B,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAQ3C,iBAAiB,EAAE,MAAM,CAAC;IAM1B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAG/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAI9B,iBAAiB,EAAE,aAAa,CAC9B,OAAO,4BAA4B,GAAG,OAAO,6BAA6B,CAC3E,CAAC;CACH;AA2PD,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAjlEhB,OAAO,yBAAyB,GAChC,OAAO,+BAA+B,GACtC,OAAO,8BAA8B,GACrC,OAAO,wBAAwB;eAC5B,MAAM;;;;;;;;;;;;;;;;;;eAsCT,OAAO,sBAAsB,GAC7B,OAAO,4BAA4B,GACnC,OAAO,oBAAoB,GAC3B,OAAO,oBAAoB;eACxB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAwaT,OAAO,mBAAmB,GAC1B,OAAO,mBAAmB,GAC1B,OAAO,sBAAsB;eAC1B,MAAM;eACN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAmQwB,MAAM,GAAG,MAAM;cAAQ,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA08DnE,CAAC"}
|
package/dist/constants.js
CHANGED
|
@@ -208,6 +208,32 @@ const RESOURCE_SOURCE_TYPES = [
|
|
|
208
208
|
RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER,
|
|
209
209
|
RESOURCE_SOURCE_TYPE_CUSTOM_CODE
|
|
210
210
|
];
|
|
211
|
+
// How often a source that lives somewhere else — a crawled website, a Drive or
|
|
212
|
+
// OneDrive folder, a file imported from either — is refreshed from it. Every
|
|
213
|
+
// source is stored with an interval, weekly unless its owner changes it; the
|
|
214
|
+
// hourly sync job only acts on it for a plan that includes automatic sync, so a
|
|
215
|
+
// Free org's sources start syncing the moment it upgrades.
|
|
216
|
+
const RESOURCE_SYNC_INTERVAL_OFF = 'off';
|
|
217
|
+
const RESOURCE_SYNC_INTERVAL_DAILY = 'daily';
|
|
218
|
+
const RESOURCE_SYNC_INTERVAL_WEEKLY = 'weekly';
|
|
219
|
+
const RESOURCE_SYNC_INTERVALS = [
|
|
220
|
+
RESOURCE_SYNC_INTERVAL_OFF,
|
|
221
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
222
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
223
|
+
];
|
|
224
|
+
const RESOURCE_SYNC_INTERVAL_DEFAULT = RESOURCE_SYNC_INTERVAL_WEEKLY;
|
|
225
|
+
const RESOURCE_SYNC_INTERVAL_MS = {
|
|
226
|
+
[RESOURCE_SYNC_INTERVAL_DAILY]: 24 * 60 * 60 * 1000,
|
|
227
|
+
[RESOURCE_SYNC_INTERVAL_WEEKLY]: 7 * 24 * 60 * 60 * 1000
|
|
228
|
+
};
|
|
229
|
+
// Sources started per hourly run. A sync is cheap to start and expensive to
|
|
230
|
+
// run — a website re-crawls every page — so the job works through a backlog
|
|
231
|
+
// over several hours rather than launching all of it at once.
|
|
232
|
+
const RESOURCE_SYNC_BATCH_SIZE = 50;
|
|
233
|
+
// How soon a source can be synced by hand again. Manual sync is open to every
|
|
234
|
+
// plan, and a website sync re-crawls every page, so it can't be a button to
|
|
235
|
+
// press in a loop.
|
|
236
|
+
const RESOURCE_SYNC_MANUAL_COOLDOWN_MS = 5 * 60 * 1000;
|
|
211
237
|
const CRAWL_RENDERER_CHEERIO = 'cheerio';
|
|
212
238
|
const CRAWL_RENDERER_PLAYWRIGHT = 'playwright';
|
|
213
239
|
const CRAWL_RENDERERS = [CRAWL_RENDERER_CHEERIO, CRAWL_RENDERER_PLAYWRIGHT];
|
|
@@ -496,8 +522,8 @@ const DEFAULT_LLM_SYSTEM_PROMPT = 'You are a helpful assistant. Answer the user
|
|
|
496
522
|
const LLM_CATALOG = [
|
|
497
523
|
{
|
|
498
524
|
provider: LLM_PROVIDER_GOOGLE,
|
|
499
|
-
model: 'gemini-3-flash
|
|
500
|
-
label: 'Gemini 3 Flash'
|
|
525
|
+
model: 'gemini-3.8-flash',
|
|
526
|
+
label: 'Gemini 3.8 Flash'
|
|
501
527
|
},
|
|
502
528
|
{
|
|
503
529
|
provider: LLM_PROVIDER_GOOGLE,
|
|
@@ -506,13 +532,13 @@ const LLM_CATALOG = [
|
|
|
506
532
|
},
|
|
507
533
|
{
|
|
508
534
|
provider: LLM_PROVIDER_OPENAI,
|
|
509
|
-
model: 'gpt-
|
|
510
|
-
label: 'GPT-
|
|
535
|
+
model: 'gpt-5.4-mini',
|
|
536
|
+
label: 'GPT-5.4 mini'
|
|
511
537
|
},
|
|
512
538
|
{
|
|
513
539
|
provider: LLM_PROVIDER_OPENAI,
|
|
514
|
-
model: 'gpt-
|
|
515
|
-
label: 'GPT-
|
|
540
|
+
model: 'gpt-5.5',
|
|
541
|
+
label: 'GPT-5.5'
|
|
516
542
|
},
|
|
517
543
|
{
|
|
518
544
|
provider: LLM_PROVIDER_ANTHROPIC,
|
|
@@ -521,15 +547,30 @@ const LLM_CATALOG = [
|
|
|
521
547
|
},
|
|
522
548
|
{
|
|
523
549
|
provider: LLM_PROVIDER_ANTHROPIC,
|
|
524
|
-
model: 'claude-sonnet-
|
|
525
|
-
label: 'Claude Sonnet
|
|
550
|
+
model: 'claude-sonnet-5',
|
|
551
|
+
label: 'Claude Sonnet 5'
|
|
526
552
|
},
|
|
527
553
|
{
|
|
528
554
|
provider: LLM_PROVIDER_ANTHROPIC,
|
|
529
|
-
model: 'claude-opus-
|
|
530
|
-
label: 'Claude
|
|
555
|
+
model: 'claude-opus-5',
|
|
556
|
+
label: 'Claude Opus 5'
|
|
531
557
|
}
|
|
532
558
|
];
|
|
559
|
+
// The picker's escape hatch. Anything that speaks the OpenAI chat-completions
|
|
560
|
+
// protocol — a gateway, an aggregator, a self-hosted server — is reachable
|
|
561
|
+
// without a catalog row: the customer supplies the model id and the base URL,
|
|
562
|
+
// and the request goes out through the OpenAI adapter. Not a provider value, so
|
|
563
|
+
// it can never reach the database; the dashboard translates it into
|
|
564
|
+
// LLM_PROVIDER_OPENAI_COMPATIBLE on submit.
|
|
565
|
+
const LLM_CATALOG_CUSTOM_KEY = 'custom';
|
|
566
|
+
// Validation messages for an organization LLM's base URL. Constants rather than
|
|
567
|
+
// inline strings because localizeZodIssue keys its translations on the exact
|
|
568
|
+
// English text, so the two files have to name the same value. Both also carry a
|
|
569
|
+
// word matchStatus recognises ("required", "must be"): the update path re-checks
|
|
570
|
+
// the merged row and throws the message as a plain Error, and anything
|
|
571
|
+
// unrecognised there becomes an opaque 500 instead of a 400.
|
|
572
|
+
const LLM_BASE_URL_REQUIRED_MESSAGE = 'A base URL is required for an OpenAI-compatible model';
|
|
573
|
+
const LLM_BASE_URL_INVALID_MESSAGE = 'Base URL must be a public https:// address';
|
|
533
574
|
const LLM_SYSTEM_DEFAULT = 'SYSTEM_DEFAULT';
|
|
534
575
|
const MAX_TOOL_LOOPS = 8;
|
|
535
576
|
// How many recent messages a channel turn replays as context.
|
|
@@ -692,6 +733,24 @@ const EMBEDDING_DIMENSIONS = 1536;
|
|
|
692
733
|
const CHUNK_TARGET_CHARS = 2000;
|
|
693
734
|
const CHUNK_OVERLAP_CHARS = 200;
|
|
694
735
|
const EMBED_BATCH_SIZE = 96;
|
|
736
|
+
// Hybrid resource search. Each ranker contributes its top candidates, and the
|
|
737
|
+
// two lists are merged with reciprocal rank fusion: score = sum of 1/(k + rank).
|
|
738
|
+
// Fusion reads ranks, never raw scores, so a cosine distance and a ts_rank never
|
|
739
|
+
// have to be put on one scale. k = 60 is the constant from the original RRF
|
|
740
|
+
// paper; it keeps one list's #1 from outweighing a chunk both lists agree on.
|
|
741
|
+
const RESOURCE_SEARCH_CANDIDATES = 50;
|
|
742
|
+
const RESOURCE_SEARCH_RRF_K = 60;
|
|
743
|
+
// A query word found in more than 1 of every 10 chunks of the artifact is
|
|
744
|
+
// dropped from the lexical query — never below 5 chunks, so a small artifact
|
|
745
|
+
// keeps its words.
|
|
746
|
+
const RESOURCE_SEARCH_COMMON_TERM_DIVISOR = 10;
|
|
747
|
+
const RESOURCE_SEARCH_COMMON_TERM_MIN = 5;
|
|
748
|
+
// Second stage, on wherever the Worker has an AI binding: the top fused
|
|
749
|
+
// candidates are re-scored by a cross-encoder, which reads query and chunk
|
|
750
|
+
// together instead of comparing two independently computed vectors. One model
|
|
751
|
+
// call per search; a failure falls back to the fused order.
|
|
752
|
+
const RESOURCE_SEARCH_RERANK_CANDIDATES = 30;
|
|
753
|
+
const RESOURCE_SEARCH_RERANK_MODEL = '@cf/baai/bge-reranker-base';
|
|
695
754
|
const RESOURCE_HANDLER_SLEEP_AFTER = '10m';
|
|
696
755
|
const DOCS_URL = 'https://docs.ganju.ai';
|
|
697
756
|
const BASE64_DATA_URI_RE = /data:image\/[a-zA-Z0-9+\-.]+;base64,[A-Za-z0-9+/=\s]+/g;
|
|
@@ -996,8 +1055,7 @@ const MCP_PROXY_TOOL_NAME_MAX = 64;
|
|
|
996
1055
|
//
|
|
997
1056
|
// The row's config holds only `activeVersionId` plus the egress/limit settings;
|
|
998
1057
|
// names and schemas live on the version. That's what keeps MCP boot off the
|
|
999
|
-
// dispatcher — a slow or broken script must never break tools/list.
|
|
1000
|
-
// docs/CUSTOM_TOOLS.md ("Boot contract").
|
|
1058
|
+
// dispatcher — a slow or broken script must never break tools/list.
|
|
1001
1059
|
const TOOL_DEFINITION_KEY_CUSTOM_CODE = 'custom-code';
|
|
1002
1060
|
// A version is uploaded as `draft`, becomes `published` when it is made active,
|
|
1003
1061
|
// and is `archived` when a later version replaces it. Exactly one version per
|
|
@@ -1677,7 +1735,8 @@ const PLAN_LIMITS = {
|
|
|
1677
1735
|
includedToolCalls: 0,
|
|
1678
1736
|
toolCallHardCap: 10_000,
|
|
1679
1737
|
includedMessages: 100,
|
|
1680
|
-
includedEmbeddedBytes: 5 * MB
|
|
1738
|
+
includedEmbeddedBytes: 5 * MB,
|
|
1739
|
+
autoSyncIntervals: []
|
|
1681
1740
|
},
|
|
1682
1741
|
PRO: {
|
|
1683
1742
|
maxProjects: null,
|
|
@@ -1700,7 +1759,11 @@ const PLAN_LIMITS = {
|
|
|
1700
1759
|
includedToolCalls: PRICING_INCLUDED_TOOL_CALLS,
|
|
1701
1760
|
toolCallHardCap: PRICING_TOOL_CALL_HARD_CAP,
|
|
1702
1761
|
includedMessages: PRICING_INCLUDED_MESSAGES,
|
|
1703
|
-
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
|
|
1762
|
+
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
|
|
1763
|
+
autoSyncIntervals: [
|
|
1764
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
1765
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
1766
|
+
]
|
|
1704
1767
|
},
|
|
1705
1768
|
ENTERPRISE: {
|
|
1706
1769
|
maxProjects: null,
|
|
@@ -1723,7 +1786,11 @@ const PLAN_LIMITS = {
|
|
|
1723
1786
|
// contract; don't make one plan's abuse limit another plan's product limit.
|
|
1724
1787
|
toolCallHardCap: null,
|
|
1725
1788
|
includedMessages: PRICING_INCLUDED_MESSAGES,
|
|
1726
|
-
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
|
|
1789
|
+
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
|
|
1790
|
+
autoSyncIntervals: [
|
|
1791
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
1792
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
1793
|
+
]
|
|
1727
1794
|
}
|
|
1728
1795
|
};
|
|
1729
1796
|
// Quota features — used as the `feature` discriminator on PlanLimitError so the
|
|
@@ -1741,6 +1808,7 @@ const PLAN_FEATURE_RAW_STORAGE = 'rawStorage';
|
|
|
1741
1808
|
const PLAN_FEATURE_EMBEDDED_STORAGE = 'embeddedStorage';
|
|
1742
1809
|
const PLAN_FEATURE_MESSAGE = 'message';
|
|
1743
1810
|
const PLAN_FEATURE_TOOL_CALL = 'toolCall';
|
|
1811
|
+
const PLAN_FEATURE_RESOURCE_SYNC = 'resourceSync';
|
|
1744
1812
|
// Stable code returned on a quota block so clients can branch on it (402).
|
|
1745
1813
|
const PLAN_LIMIT_ERROR_CODE = 'PLAN_LIMIT_EXCEEDED';
|
|
1746
1814
|
// Billing meter event names. The metering cron reports per-period OVERAGE
|
|
@@ -1783,7 +1851,7 @@ const POLAR_SERVER_PRODUCTION = 'production';
|
|
|
1783
1851
|
const CONSENT_DOCUMENT_TERMS = 'terms';
|
|
1784
1852
|
const CONSENT_DOCUMENT_PRIVACY = 'privacy';
|
|
1785
1853
|
const CONSENT_DOCUMENTS = [CONSENT_DOCUMENT_TERMS, CONSENT_DOCUMENT_PRIVACY];
|
|
1786
|
-
const CONSENT_CURRENT_VERSION = '2026-
|
|
1854
|
+
const CONSENT_CURRENT_VERSION = '2026-09-10';
|
|
1787
1855
|
const CONSENT_SOURCE_SIGNUP = 'signup';
|
|
1788
1856
|
const CONSENT_SOURCE_REACCEPT = 'reaccept';
|
|
1789
1857
|
const CONSENT_SOURCES = [CONSENT_SOURCE_SIGNUP, CONSENT_SOURCE_REACCEPT];
|
|
@@ -1908,6 +1976,7 @@ exports.constants = {
|
|
|
1908
1976
|
PLAN_FEATURE_EMBEDDED_STORAGE,
|
|
1909
1977
|
PLAN_FEATURE_MESSAGE,
|
|
1910
1978
|
PLAN_FEATURE_TOOL_CALL,
|
|
1979
|
+
PLAN_FEATURE_RESOURCE_SYNC,
|
|
1911
1980
|
PLAN_LIMIT_ERROR_CODE,
|
|
1912
1981
|
BILLING_METER_MESSAGES,
|
|
1913
1982
|
BILLING_METER_SHARED_MESSAGES,
|
|
@@ -2102,6 +2171,14 @@ exports.constants = {
|
|
|
2102
2171
|
RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER,
|
|
2103
2172
|
RESOURCE_SOURCE_TYPE_CUSTOM_CODE,
|
|
2104
2173
|
RESOURCE_SOURCE_TYPES,
|
|
2174
|
+
RESOURCE_SYNC_INTERVAL_OFF,
|
|
2175
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
2176
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY,
|
|
2177
|
+
RESOURCE_SYNC_INTERVALS,
|
|
2178
|
+
RESOURCE_SYNC_INTERVAL_DEFAULT,
|
|
2179
|
+
RESOURCE_SYNC_INTERVAL_MS,
|
|
2180
|
+
RESOURCE_SYNC_BATCH_SIZE,
|
|
2181
|
+
RESOURCE_SYNC_MANUAL_COOLDOWN_MS,
|
|
2105
2182
|
CRAWL_RENDERER_CHEERIO,
|
|
2106
2183
|
CRAWL_RENDERER_PLAYWRIGHT,
|
|
2107
2184
|
CRAWL_RENDERERS,
|
|
@@ -2151,6 +2228,9 @@ exports.constants = {
|
|
|
2151
2228
|
DEFAULT_LLM_MODEL,
|
|
2152
2229
|
DEFAULT_LLM_SYSTEM_PROMPT,
|
|
2153
2230
|
LLM_CATALOG,
|
|
2231
|
+
LLM_CATALOG_CUSTOM_KEY,
|
|
2232
|
+
LLM_BASE_URL_REQUIRED_MESSAGE,
|
|
2233
|
+
LLM_BASE_URL_INVALID_MESSAGE,
|
|
2154
2234
|
LLM_SYSTEM_DEFAULT,
|
|
2155
2235
|
MAX_TOOL_LOOPS,
|
|
2156
2236
|
CHANNEL_HISTORY_LIMIT,
|
|
@@ -2220,6 +2300,12 @@ exports.constants = {
|
|
|
2220
2300
|
CHUNK_TARGET_CHARS,
|
|
2221
2301
|
CHUNK_OVERLAP_CHARS,
|
|
2222
2302
|
EMBED_BATCH_SIZE,
|
|
2303
|
+
RESOURCE_SEARCH_CANDIDATES,
|
|
2304
|
+
RESOURCE_SEARCH_RRF_K,
|
|
2305
|
+
RESOURCE_SEARCH_COMMON_TERM_DIVISOR,
|
|
2306
|
+
RESOURCE_SEARCH_COMMON_TERM_MIN,
|
|
2307
|
+
RESOURCE_SEARCH_RERANK_CANDIDATES,
|
|
2308
|
+
RESOURCE_SEARCH_RERANK_MODEL,
|
|
2223
2309
|
RESOURCE_HANDLER_SLEEP_AFTER,
|
|
2224
2310
|
DOCS_URL,
|
|
2225
2311
|
BASE64_DATA_URI_RE,
|