@ganju/utils 0.0.9 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channelDebounce.d.ts +11 -0
- package/dist/channelDebounce.d.ts.map +1 -1
- package/dist/channelDebounce.js +12 -1
- package/dist/chunking.d.ts +17 -3
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +125 -19
- package/dist/constants.d.ts +21 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +73 -5
- package/dist/index.d.ts +68 -25
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -0
- package/dist/resourceRerank.d.ts +20 -0
- package/dist/resourceRerank.d.ts.map +1 -0
- package/dist/resourceRerank.js +37 -0
- package/dist/schema.d.ts +39 -22
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +15 -0
- package/dist/syncableResource.d.ts +9 -0
- package/dist/syncableResource.d.ts.map +1 -0
- package/dist/syncableResource.js +31 -0
- package/dist/toolCatalog.d.ts +1 -1
- package/dist/toolCatalog.js +1 -1
- package/package.json +1 -1
- package/src/channelDebounce.ts +36 -0
- package/src/chunking.ts +166 -23
- package/src/constants.ts +81 -5
- package/src/index.ts +18 -1
- package/src/resourceRerank.ts +54 -0
- package/src/schema.ts +17 -0
- package/src/syncableResource.ts +48 -0
- package/src/toolCatalog.ts +1 -1
|
@@ -2,7 +2,17 @@ export interface BufferedChannelMessage {
|
|
|
2
2
|
text: string;
|
|
3
3
|
externalMessageId: string | null;
|
|
4
4
|
receivedAt: number;
|
|
5
|
+
immediate?: boolean;
|
|
6
|
+
command?: BufferedChannelCommand | null;
|
|
7
|
+
reply?: Record<string, unknown> | null;
|
|
5
8
|
}
|
|
9
|
+
export interface BufferedChannelCommand {
|
|
10
|
+
promptId: string;
|
|
11
|
+
artifactPromptId: string | null;
|
|
12
|
+
promptTitle: string;
|
|
13
|
+
args: Record<string, string>;
|
|
14
|
+
}
|
|
15
|
+
export declare const batchCommand: (messages: BufferedChannelMessage[]) => BufferedChannelCommand | null;
|
|
6
16
|
export interface ChannelBufferEnvelope {
|
|
7
17
|
channelId: string;
|
|
8
18
|
platform: string;
|
|
@@ -17,6 +27,7 @@ export interface ChannelBufferEnvelope {
|
|
|
17
27
|
export interface ChannelBufferFlush {
|
|
18
28
|
envelope: ChannelBufferEnvelope;
|
|
19
29
|
messages: BufferedChannelMessage[];
|
|
30
|
+
batchId: string;
|
|
20
31
|
}
|
|
21
32
|
export declare const resolveDebounceMs: (config: unknown) => number;
|
|
22
33
|
export declare const channelBufferKey: (channelId: string, externalConversationId: string, externalParticipantId: string) => string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"channelDebounce.d.ts","sourceRoot":"","sources":["../src/channelDebounce.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"channelDebounce.d.ts","sourceRoot":"","sources":["../src/channelDebounce.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IAKnB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,OAAO,CAAC,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAKxC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACxC;AAID,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAID,eAAO,MAAM,YAAY,GACvB,UAAU,sBAAsB,EAAE,KACjC,sBAAsB,GAAG,IAM3B,CAAC;AAOF,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACpD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAGD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,qBAAqB,CAAC;IAChC,QAAQ,EAAE,sBAAsB,EAAE,CAAC;IAGnC,OAAO,EAAE,MAAM,CAAC;CACjB;AAOD,eAAO,MAAM,iBAAiB,GAAI,QAAQ,OAAO,KAAG,MAYnD,CAAC;AAKF,eAAO,MAAM,gBAAgB,GAC3B,WAAW,MAAM,EACjB,wBAAwB,MAAM,EAC9B,uBAAuB,MAAM,KAC5B,MAA2E,CAAC;AAI/E,eAAO,MAAM,oBAAoB,GAC/B,UAAU,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,KAChC,MAIuC,CAAC"}
|
package/dist/channelDebounce.js
CHANGED
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.joinBufferedMessages = exports.channelBufferKey = exports.resolveDebounceMs = void 0;
|
|
3
|
+
exports.joinBufferedMessages = exports.channelBufferKey = exports.resolveDebounceMs = exports.batchCommand = void 0;
|
|
4
4
|
const constants_1 = require("./constants");
|
|
5
|
+
// The prompt a batch runs: the newest command in it, since that is the one the
|
|
6
|
+
// participant asked for last. Null for a batch of plain text.
|
|
7
|
+
const batchCommand = (messages) => {
|
|
8
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
9
|
+
const command = messages[i].command;
|
|
10
|
+
if (command)
|
|
11
|
+
return command;
|
|
12
|
+
}
|
|
13
|
+
return null;
|
|
14
|
+
};
|
|
15
|
+
exports.batchCommand = batchCommand;
|
|
5
16
|
// Read the channel's debounce window off its `config`, clamped to the supported
|
|
6
17
|
// range. An unset value takes the default; an explicit 0 disables buffering, so
|
|
7
18
|
// the channel answers every message as it arrives. Anything malformed (a string,
|
package/dist/chunking.d.ts
CHANGED
|
@@ -6,6 +6,17 @@ export declare const chunkText: (text: string, options?: {
|
|
|
6
6
|
targetChars?: number;
|
|
7
7
|
overlapChars?: number;
|
|
8
8
|
}) => string[];
|
|
9
|
+
export type HeadingSyntax = 'markdown' | 'html';
|
|
10
|
+
/**
|
|
11
|
+
* Which heading syntax a document is written in, or null when it has none we
|
|
12
|
+
* can trust. Plain text is left out on purpose: a line that happens to start
|
|
13
|
+
* with "# " there is as likely a comment or a list as a title.
|
|
14
|
+
*/
|
|
15
|
+
export declare const headingSyntaxFor: (input: {
|
|
16
|
+
mimeType?: string | null;
|
|
17
|
+
fileName?: string | null;
|
|
18
|
+
isWebPage?: boolean;
|
|
19
|
+
}) => HeadingSyntax | null;
|
|
9
20
|
export declare const buildHeader: (input: {
|
|
10
21
|
title?: string | null;
|
|
11
22
|
description?: string | null;
|
|
@@ -13,9 +24,10 @@ export declare const buildHeader: (input: {
|
|
|
13
24
|
mimeType?: string | null;
|
|
14
25
|
fileName?: string | null;
|
|
15
26
|
}) => string;
|
|
16
|
-
export interface ChunkMetadata extends ExtractedDocumentMetadata {
|
|
17
|
-
chunkIndexInPage
|
|
18
|
-
chunksInPage
|
|
27
|
+
export interface ChunkMetadata extends Partial<ExtractedDocumentMetadata> {
|
|
28
|
+
chunkIndexInPage?: number;
|
|
29
|
+
chunksInPage?: number;
|
|
30
|
+
headingPath?: string[];
|
|
19
31
|
}
|
|
20
32
|
export interface PreparedChunk {
|
|
21
33
|
content: string;
|
|
@@ -24,5 +36,7 @@ export interface PreparedChunk {
|
|
|
24
36
|
export declare const prepareChunks: (header: string, documents: ExtractedDocument[] | null, fallbackContent: string | null, options?: {
|
|
25
37
|
targetChars?: number;
|
|
26
38
|
overlapChars?: number;
|
|
39
|
+
mimeType?: string | null;
|
|
40
|
+
fileName?: string | null;
|
|
27
41
|
}) => PreparedChunk[];
|
|
28
42
|
//# sourceMappingURL=chunking.d.ts.map
|
package/dist/chunking.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,KAAK,EACV,iBAAiB,EACjB,yBAAyB,EAC1B,MAAM,qBAAqB,CAAC;AAE7B,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,SAAS,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AASpE,eAAO,MAAM,cAAc,GACzB,MAAM,MAAM,EACZ,QAAQ,MAAM,EACd,aAAY,SAAS,SAAS,EAA+B,KAC5D,MAAM,EA8BR,CAAC;AA4BF,eAAO,MAAM,SAAS,GACpB,MAAM,MAAM,EACZ,UAAU;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KACxD,MAAM,EAAoD,CAAC;AAE9D,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,MAAM,CAAC;AAehD;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,GAAI,OAAO;IACtC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,KAAG,aAAa,GAAG,IAUnB,CAAC;AAuEF,eAAO,MAAM,WAAW,GAAI,OAAO;IACjC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KAAG,MASW,CAAC;AAIhB,MAAM,WAAW,aAAc,SAAQ,OAAO,CAAC,yBAAyB,CAAC;IACvE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CAChC;AAID,eAAO,MAAM,aAAa,GACxB,QAAQ,MAAM,EACd,WAAW,iBAAiB,EAAE,GAAG,IAAI,EACrC,iBAAiB,MAAM,GAAG,IAAI,EAC9B,UAAU;IACR,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KACA,aAAa,EA4Cf,CAAC"}
|
package/dist/chunking.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.prepareChunks = exports.buildHeader = exports.chunkText = exports.splitRecursive = void 0;
|
|
3
|
+
exports.prepareChunks = exports.buildHeader = exports.headingSyntaxFor = exports.chunkText = exports.splitRecursive = void 0;
|
|
4
4
|
const constants_1 = require("./constants");
|
|
5
|
+
const sanitize_1 = require("./sanitize");
|
|
5
6
|
const trySplit = (text, sep) => {
|
|
6
7
|
if (typeof sep === 'string') {
|
|
7
8
|
return text.includes(sep) ? text.split(sep) : null;
|
|
@@ -40,23 +41,115 @@ const splitRecursive = (text, target, separators = constants_1.constants.CHUNK_S
|
|
|
40
41
|
return chunks;
|
|
41
42
|
};
|
|
42
43
|
exports.splitRecursive = splitRecursive;
|
|
43
|
-
|
|
44
|
+
// Each piece keeps its `raw` text — the slice of the input it covers, without
|
|
45
|
+
// the overlap borrowed from the piece before — so a caller can walk the input
|
|
46
|
+
// in order without reading any of it twice.
|
|
47
|
+
const chunkPieces = (text, options) => {
|
|
44
48
|
const target = options?.targetChars ?? constants_1.constants.CHUNK_TARGET_CHARS;
|
|
45
49
|
const overlap = options?.overlapChars ?? constants_1.constants.CHUNK_OVERLAP_CHARS;
|
|
46
50
|
const trimmed = text.trim();
|
|
47
51
|
if (!trimmed)
|
|
48
52
|
return [];
|
|
49
53
|
if (trimmed.length <= target)
|
|
50
|
-
return [trimmed];
|
|
54
|
+
return [{ raw: trimmed, content: trimmed }];
|
|
51
55
|
const rawChunks = (0, exports.splitRecursive)(trimmed, target);
|
|
52
|
-
const
|
|
56
|
+
const pieces = [];
|
|
53
57
|
for (let i = 0; i < rawChunks.length; i++) {
|
|
54
58
|
const prev = i > 0 ? rawChunks[i - 1].slice(-overlap) : '';
|
|
55
|
-
|
|
59
|
+
pieces.push({
|
|
60
|
+
raw: rawChunks[i],
|
|
61
|
+
content: (prev ? prev + '\n' : '') + rawChunks[i]
|
|
62
|
+
});
|
|
56
63
|
}
|
|
57
|
-
return
|
|
64
|
+
return pieces.filter(p => p.content.trim().length > 0);
|
|
58
65
|
};
|
|
66
|
+
const chunkText = (text, options) => chunkPieces(text, options).map(p => p.content);
|
|
59
67
|
exports.chunkText = chunkText;
|
|
68
|
+
const MARKDOWN_HEADING = /^ {0,3}(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/;
|
|
69
|
+
const MARKDOWN_FENCE = /^ {0,3}(```|~~~)/;
|
|
70
|
+
const HTML_HEADING = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi;
|
|
71
|
+
const MAX_HEADING_CHARS = 120;
|
|
72
|
+
const MARKDOWN_FILE_NAME = /\.(md|markdown|mdx)$/i;
|
|
73
|
+
/**
|
|
74
|
+
* Which heading syntax a document is written in, or null when it has none we
|
|
75
|
+
* can trust. Plain text is left out on purpose: a line that happens to start
|
|
76
|
+
* with "# " there is as likely a comment or a list as a title.
|
|
77
|
+
*/
|
|
78
|
+
const headingSyntaxFor = (input) => {
|
|
79
|
+
// Crawled pages are flattened to text with their headings written back as
|
|
80
|
+
// Markdown lines, whatever the page's own content type was.
|
|
81
|
+
if (input.isWebPage)
|
|
82
|
+
return 'markdown';
|
|
83
|
+
if (input.mimeType === constants_1.constants.MIMETYPE_TEXT_MARKDOWN)
|
|
84
|
+
return 'markdown';
|
|
85
|
+
if (input.mimeType === constants_1.constants.MIMETYPE_TEXT_HTML)
|
|
86
|
+
return 'html';
|
|
87
|
+
if (input.fileName && MARKDOWN_FILE_NAME.test(input.fileName)) {
|
|
88
|
+
return 'markdown';
|
|
89
|
+
}
|
|
90
|
+
return null;
|
|
91
|
+
};
|
|
92
|
+
exports.headingSyntaxFor = headingSyntaxFor;
|
|
93
|
+
const cleanHeading = (text) => text.replace(/\s+/g, ' ').trim().slice(0, MAX_HEADING_CHARS);
|
|
94
|
+
const scanMarkdownHeadings = (text, state) => {
|
|
95
|
+
const events = [];
|
|
96
|
+
let offset = 0;
|
|
97
|
+
for (const line of text.split('\n')) {
|
|
98
|
+
// A `#` inside a fenced block is a shell comment or a preprocessor line,
|
|
99
|
+
// not a title — and a fence can open in one chunk and close in the next.
|
|
100
|
+
if (MARKDOWN_FENCE.test(line)) {
|
|
101
|
+
state.inFence = !state.inFence;
|
|
102
|
+
}
|
|
103
|
+
else if (!state.inFence) {
|
|
104
|
+
const match = MARKDOWN_HEADING.exec(line);
|
|
105
|
+
const heading = match ? cleanHeading(match[2]) : '';
|
|
106
|
+
if (match && heading) {
|
|
107
|
+
events.push({ offset, level: match[1].length, text: heading });
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
offset += line.length + 1;
|
|
111
|
+
}
|
|
112
|
+
return events;
|
|
113
|
+
};
|
|
114
|
+
const scanHtmlHeadings = (text) => [...text.matchAll(HTML_HEADING)]
|
|
115
|
+
.map(match => ({
|
|
116
|
+
offset: match.index ?? 0,
|
|
117
|
+
level: Number(match[1]),
|
|
118
|
+
text: cleanHeading((0, sanitize_1.decodeEntities)(match[2].replace(/<[^>]*>/g, ' ')))
|
|
119
|
+
}))
|
|
120
|
+
.filter(event => event.text);
|
|
121
|
+
/**
|
|
122
|
+
* Follows the heading hierarchy across a document's chunks, fed in order.
|
|
123
|
+
* Returns, for each chunk, the path of headings it sits under — the section a
|
|
124
|
+
* reader would be pointed to. A chunk that opens with a heading belongs to that
|
|
125
|
+
* heading; one that starts mid-section belongs to the section it continues,
|
|
126
|
+
* even when a later heading begins inside it.
|
|
127
|
+
*/
|
|
128
|
+
const createHeadingTracker = (syntax) => {
|
|
129
|
+
let stack = [];
|
|
130
|
+
const fence = { inFence: false };
|
|
131
|
+
const apply = (event) => {
|
|
132
|
+
stack = stack.filter(h => h.level < event.level);
|
|
133
|
+
stack.push({ level: event.level, text: event.text });
|
|
134
|
+
};
|
|
135
|
+
return (raw) => {
|
|
136
|
+
const events = syntax === 'markdown'
|
|
137
|
+
? scanMarkdownHeadings(raw, fence)
|
|
138
|
+
: scanHtmlHeadings(raw);
|
|
139
|
+
const [first, ...rest] = events;
|
|
140
|
+
// Before a document's first heading there is no section to continue — only
|
|
141
|
+
// the metadata header, or the markup around an HTML body — so a chunk that
|
|
142
|
+
// holds the first heading belongs to it wherever in the chunk it falls.
|
|
143
|
+
const opensWithHeading = first !== undefined &&
|
|
144
|
+
(stack.length === 0 || raw.slice(0, first.offset).trim() === '');
|
|
145
|
+
if (opensWithHeading)
|
|
146
|
+
apply(first);
|
|
147
|
+
const path = stack.map(h => h.text);
|
|
148
|
+
for (const event of opensWithHeading ? rest : events)
|
|
149
|
+
apply(event);
|
|
150
|
+
return path;
|
|
151
|
+
};
|
|
152
|
+
};
|
|
60
153
|
const buildHeader = (input) => [
|
|
61
154
|
input.title ? `Title: ${input.title}` : null,
|
|
62
155
|
input.description ? `Description: ${input.description}` : null,
|
|
@@ -67,37 +160,50 @@ const buildHeader = (input) => [
|
|
|
67
160
|
.filter(Boolean)
|
|
68
161
|
.join('\n');
|
|
69
162
|
exports.buildHeader = buildHeader;
|
|
163
|
+
const CONTROL_CHARS = /[\x00-\x08\x0B\x0C\x0E-\x1F]/g;
|
|
70
164
|
const prepareChunks = (header, documents, fallbackContent, options) => {
|
|
71
165
|
if (documents && documents.length > 0) {
|
|
72
166
|
const out = [];
|
|
73
167
|
documents.forEach((doc, docIndex) => {
|
|
74
|
-
const body = (doc.pageContent || '')
|
|
75
|
-
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
|
|
76
|
-
.trim();
|
|
168
|
+
const body = (doc.pageContent || '').replace(CONTROL_CHARS, '').trim();
|
|
77
169
|
if (!body)
|
|
78
170
|
return;
|
|
79
171
|
const text = docIndex === 0 && header ? `${header}\n\n${body}` : body;
|
|
80
|
-
const
|
|
172
|
+
const syntax = (0, exports.headingSyntaxFor)({
|
|
173
|
+
mimeType: doc.metadata?.source?.mimeType ?? options?.mimeType,
|
|
174
|
+
fileName: doc.metadata?.source?.fileName ?? options?.fileName,
|
|
175
|
+
isWebPage: !!doc.metadata?.web
|
|
176
|
+
});
|
|
177
|
+
const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
|
|
178
|
+
const pieces = chunkPieces(text, options);
|
|
81
179
|
pieces.forEach((piece, pieceIndex) => {
|
|
180
|
+
const headingPath = headingPathOf?.(piece.raw) ?? [];
|
|
82
181
|
out.push({
|
|
83
|
-
content: piece,
|
|
182
|
+
content: piece.content,
|
|
84
183
|
metadata: {
|
|
85
184
|
...doc.metadata,
|
|
86
185
|
chunkIndexInPage: pieceIndex,
|
|
87
|
-
chunksInPage: pieces.length
|
|
186
|
+
chunksInPage: pieces.length,
|
|
187
|
+
...(headingPath.length > 0 ? { headingPath } : {})
|
|
88
188
|
}
|
|
89
189
|
});
|
|
90
190
|
});
|
|
91
191
|
});
|
|
92
192
|
return out;
|
|
93
193
|
}
|
|
94
|
-
const body = (fallbackContent || '')
|
|
95
|
-
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
|
|
96
|
-
.trim();
|
|
194
|
+
const body = (fallbackContent || '').replace(CONTROL_CHARS, '').trim();
|
|
97
195
|
const fullText = [header, body].filter(Boolean).join('\n\n');
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
})
|
|
196
|
+
const syntax = (0, exports.headingSyntaxFor)({
|
|
197
|
+
mimeType: options?.mimeType,
|
|
198
|
+
fileName: options?.fileName
|
|
199
|
+
});
|
|
200
|
+
const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
|
|
201
|
+
return chunkPieces(fullText, options).map(piece => {
|
|
202
|
+
const headingPath = headingPathOf?.(piece.raw) ?? [];
|
|
203
|
+
return {
|
|
204
|
+
content: piece.content,
|
|
205
|
+
metadata: headingPath.length > 0 ? { headingPath } : null
|
|
206
|
+
};
|
|
207
|
+
});
|
|
102
208
|
};
|
|
103
209
|
exports.prepareChunks = prepareChunks;
|
package/dist/constants.d.ts
CHANGED
|
@@ -6,6 +6,8 @@ declare const ONE_DRIVE_TAB_MY_FILES: "my-files";
|
|
|
6
6
|
declare const ONE_DRIVE_TAB_SHARED_WITH_ME: "shared-with-me";
|
|
7
7
|
declare const ONE_DRIVE_TAB_RECENT: "recent";
|
|
8
8
|
declare const ONE_DRIVE_TAB_DRIVES: "drives";
|
|
9
|
+
declare const RESOURCE_SYNC_INTERVAL_DAILY: "daily";
|
|
10
|
+
declare const RESOURCE_SYNC_INTERVAL_WEEKLY: "weekly";
|
|
9
11
|
declare const LLM_PROVIDER_ANTHROPIC: "anthropic";
|
|
10
12
|
declare const LLM_PROVIDER_OPENAI: "openai";
|
|
11
13
|
declare const LLM_PROVIDER_GOOGLE: "google";
|
|
@@ -59,6 +61,7 @@ interface PlanLimits {
|
|
|
59
61
|
toolCallHardCap: number | null;
|
|
60
62
|
includedMessages: number;
|
|
61
63
|
includedEmbeddedBytes: number;
|
|
64
|
+
autoSyncIntervals: ReadonlyArray<typeof RESOURCE_SYNC_INTERVAL_DAILY | typeof RESOURCE_SYNC_INTERVAL_WEEKLY>;
|
|
62
65
|
}
|
|
63
66
|
export type { PlanLimits };
|
|
64
67
|
export declare const constants: {
|
|
@@ -104,6 +107,7 @@ export declare const constants: {
|
|
|
104
107
|
PLAN_FEATURE_EMBEDDED_STORAGE: "embeddedStorage";
|
|
105
108
|
PLAN_FEATURE_MESSAGE: "message";
|
|
106
109
|
PLAN_FEATURE_TOOL_CALL: "toolCall";
|
|
110
|
+
PLAN_FEATURE_RESOURCE_SYNC: "resourceSync";
|
|
107
111
|
PLAN_LIMIT_ERROR_CODE: string;
|
|
108
112
|
BILLING_METER_MESSAGES: string;
|
|
109
113
|
BILLING_METER_SHARED_MESSAGES: string;
|
|
@@ -309,6 +313,17 @@ export declare const constants: {
|
|
|
309
313
|
RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER: "ONE_DRIVE_FOLDER";
|
|
310
314
|
RESOURCE_SOURCE_TYPE_CUSTOM_CODE: "CUSTOM_CODE";
|
|
311
315
|
RESOURCE_SOURCE_TYPES: ("FILE" | "WEBSITE" | "GOOGLE_DRIVE_FOLDER" | "ONE_DRIVE_FOLDER" | "CUSTOM_CODE")[];
|
|
316
|
+
RESOURCE_SYNC_INTERVAL_OFF: "off";
|
|
317
|
+
RESOURCE_SYNC_INTERVAL_DAILY: "daily";
|
|
318
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY: "weekly";
|
|
319
|
+
RESOURCE_SYNC_INTERVALS: readonly ["off", "daily", "weekly"];
|
|
320
|
+
RESOURCE_SYNC_INTERVAL_DEFAULT: "weekly";
|
|
321
|
+
RESOURCE_SYNC_INTERVAL_MS: {
|
|
322
|
+
readonly daily: number;
|
|
323
|
+
readonly weekly: number;
|
|
324
|
+
};
|
|
325
|
+
RESOURCE_SYNC_BATCH_SIZE: number;
|
|
326
|
+
RESOURCE_SYNC_MANUAL_COOLDOWN_MS: number;
|
|
312
327
|
CRAWL_RENDERER_CHEERIO: "cheerio";
|
|
313
328
|
CRAWL_RENDERER_PLAYWRIGHT: "playwright";
|
|
314
329
|
CRAWL_RENDERERS: ("cheerio" | "playwright")[];
|
|
@@ -434,6 +449,12 @@ export declare const constants: {
|
|
|
434
449
|
CHUNK_TARGET_CHARS: number;
|
|
435
450
|
CHUNK_OVERLAP_CHARS: number;
|
|
436
451
|
EMBED_BATCH_SIZE: number;
|
|
452
|
+
RESOURCE_SEARCH_CANDIDATES: number;
|
|
453
|
+
RESOURCE_SEARCH_RRF_K: number;
|
|
454
|
+
RESOURCE_SEARCH_COMMON_TERM_DIVISOR: number;
|
|
455
|
+
RESOURCE_SEARCH_COMMON_TERM_MIN: number;
|
|
456
|
+
RESOURCE_SEARCH_RERANK_CANDIDATES: number;
|
|
457
|
+
RESOURCE_SEARCH_RERANK_MODEL: string;
|
|
437
458
|
RESOURCE_HANDLER_SLEEP_AFTER: string;
|
|
438
459
|
DOCS_URL: string;
|
|
439
460
|
BASE64_DATA_URI_RE: RegExp;
|
package/dist/constants.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AA6JA,QAAA,MAAM,yBAAyB,EAAiB,UAAU,CAAC;AAC3D,QAAA,MAAM,+BAA+B,EAAuB,gBAAgB,CAAC;AAC7E,QAAA,MAAM,8BAA8B,EAAsB,eAAe,CAAC;AAC1E,QAAA,MAAM,wBAAwB,EAAgB,SAAS,CAAC;AAuCxD,QAAA,MAAM,sBAAsB,EAAiB,UAAU,CAAC;AACxD,QAAA,MAAM,4BAA4B,EAAuB,gBAAgB,CAAC;AAC1E,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAClD,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AA6JA,QAAA,MAAM,yBAAyB,EAAiB,UAAU,CAAC;AAC3D,QAAA,MAAM,+BAA+B,EAAuB,gBAAgB,CAAC;AAC7E,QAAA,MAAM,8BAA8B,EAAsB,eAAe,CAAC;AAC1E,QAAA,MAAM,wBAAwB,EAAgB,SAAS,CAAC;AAuCxD,QAAA,MAAM,sBAAsB,EAAiB,UAAU,CAAC;AACxD,QAAA,MAAM,4BAA4B,EAAuB,gBAAgB,CAAC;AAC1E,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAClD,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAoFlD,QAAA,MAAM,4BAA4B,EAAc,OAAO,CAAC;AACxD,QAAA,MAAM,6BAA6B,EAAe,QAAQ,CAAC;AAyU3D,QAAA,MAAM,sBAAsB,EAAkB,WAAW,CAAC;AAC1D,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAGjD,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAwVjD,MAAM,MAAM,mBAAmB,GAC3B;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC9D;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA0+BpE,UAAU,UAAU;IAElB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAGhC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAMjC,sBAAsB,EAAE,MAAM,CAAC;IAM/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,SAAS,EAAE,OAAO,CAAC;IAKnB,eAAe,EAAE,OAAO,CAAC;IAOzB,gBAAgB,EAAE,OAAO,CAAC;IAK1B,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAQ3C,iBAAiB,EAAE,MAAM,CAAC;IAM1B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAG/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAI9B,iBAAiB,EAAE,aAAa,CAC9B,OAAO,4BAA4B,GAAG,OAAO,6BAA6B,CAC3E,CAAC;CACH;AA2PD,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAjlEhB,OAAO,yBAAyB,GAChC,OAAO,+BAA+B,GACtC,OAAO,8BAA8B,GACrC,OAAO,wBAAwB;eAC5B,MAAM;;;;;;;;;;;;;;;;;;eAsCT,OAAO,sBAAsB,GAC7B,OAAO,4BAA4B,GACnC,OAAO,oBAAoB,GAC3B,OAAO,oBAAoB;eACxB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAwaT,OAAO,mBAAmB,GAC1B,OAAO,mBAAmB,GAC1B,OAAO,sBAAsB;eAC1B,MAAM;eACN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAmQwB,MAAM,GAAG,MAAM;cAAQ,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA08DnE,CAAC"}
|
package/dist/constants.js
CHANGED
|
@@ -208,6 +208,32 @@ const RESOURCE_SOURCE_TYPES = [
|
|
|
208
208
|
RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER,
|
|
209
209
|
RESOURCE_SOURCE_TYPE_CUSTOM_CODE
|
|
210
210
|
];
|
|
211
|
+
// How often a source that lives somewhere else — a crawled website, a Drive or
|
|
212
|
+
// OneDrive folder, a file imported from either — is refreshed from it. Every
|
|
213
|
+
// source is stored with an interval, weekly unless its owner changes it; the
|
|
214
|
+
// hourly sync job only acts on it for a plan that includes automatic sync, so a
|
|
215
|
+
// Free org's sources start syncing the moment it upgrades.
|
|
216
|
+
const RESOURCE_SYNC_INTERVAL_OFF = 'off';
|
|
217
|
+
const RESOURCE_SYNC_INTERVAL_DAILY = 'daily';
|
|
218
|
+
const RESOURCE_SYNC_INTERVAL_WEEKLY = 'weekly';
|
|
219
|
+
const RESOURCE_SYNC_INTERVALS = [
|
|
220
|
+
RESOURCE_SYNC_INTERVAL_OFF,
|
|
221
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
222
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
223
|
+
];
|
|
224
|
+
const RESOURCE_SYNC_INTERVAL_DEFAULT = RESOURCE_SYNC_INTERVAL_WEEKLY;
|
|
225
|
+
const RESOURCE_SYNC_INTERVAL_MS = {
|
|
226
|
+
[RESOURCE_SYNC_INTERVAL_DAILY]: 24 * 60 * 60 * 1000,
|
|
227
|
+
[RESOURCE_SYNC_INTERVAL_WEEKLY]: 7 * 24 * 60 * 60 * 1000
|
|
228
|
+
};
|
|
229
|
+
// Sources started per hourly run. A sync is cheap to start and expensive to
|
|
230
|
+
// run — a website re-crawls every page — so the job works through a backlog
|
|
231
|
+
// over several hours rather than launching all of it at once.
|
|
232
|
+
const RESOURCE_SYNC_BATCH_SIZE = 50;
|
|
233
|
+
// How soon a source can be synced by hand again. Manual sync is open to every
|
|
234
|
+
// plan, and a website sync re-crawls every page, so it can't be a button to
|
|
235
|
+
// press in a loop.
|
|
236
|
+
const RESOURCE_SYNC_MANUAL_COOLDOWN_MS = 5 * 60 * 1000;
|
|
211
237
|
const CRAWL_RENDERER_CHEERIO = 'cheerio';
|
|
212
238
|
const CRAWL_RENDERER_PLAYWRIGHT = 'playwright';
|
|
213
239
|
const CRAWL_RENDERERS = [CRAWL_RENDERER_CHEERIO, CRAWL_RENDERER_PLAYWRIGHT];
|
|
@@ -707,6 +733,24 @@ const EMBEDDING_DIMENSIONS = 1536;
|
|
|
707
733
|
const CHUNK_TARGET_CHARS = 2000;
|
|
708
734
|
const CHUNK_OVERLAP_CHARS = 200;
|
|
709
735
|
const EMBED_BATCH_SIZE = 96;
|
|
736
|
+
// Hybrid resource search. Each ranker contributes its top candidates, and the
|
|
737
|
+
// two lists are merged with reciprocal rank fusion: score = sum of 1/(k + rank).
|
|
738
|
+
// Fusion reads ranks, never raw scores, so a cosine distance and a ts_rank never
|
|
739
|
+
// have to be put on one scale. k = 60 is the constant from the original RRF
|
|
740
|
+
// paper; it keeps one list's #1 from outweighing a chunk both lists agree on.
|
|
741
|
+
const RESOURCE_SEARCH_CANDIDATES = 50;
|
|
742
|
+
const RESOURCE_SEARCH_RRF_K = 60;
|
|
743
|
+
// A query word found in more than 1 of every 10 chunks of the artifact is
|
|
744
|
+
// dropped from the lexical query — never below 5 chunks, so a small artifact
|
|
745
|
+
// keeps its words.
|
|
746
|
+
const RESOURCE_SEARCH_COMMON_TERM_DIVISOR = 10;
|
|
747
|
+
const RESOURCE_SEARCH_COMMON_TERM_MIN = 5;
|
|
748
|
+
// Second stage, on wherever the Worker has an AI binding: the top fused
|
|
749
|
+
// candidates are re-scored by a cross-encoder, which reads query and chunk
|
|
750
|
+
// together instead of comparing two independently computed vectors. One model
|
|
751
|
+
// call per search; a failure falls back to the fused order.
|
|
752
|
+
const RESOURCE_SEARCH_RERANK_CANDIDATES = 30;
|
|
753
|
+
const RESOURCE_SEARCH_RERANK_MODEL = '@cf/baai/bge-reranker-base';
|
|
710
754
|
const RESOURCE_HANDLER_SLEEP_AFTER = '10m';
|
|
711
755
|
const DOCS_URL = 'https://docs.ganju.ai';
|
|
712
756
|
const BASE64_DATA_URI_RE = /data:image\/[a-zA-Z0-9+\-.]+;base64,[A-Za-z0-9+/=\s]+/g;
|
|
@@ -1011,8 +1055,7 @@ const MCP_PROXY_TOOL_NAME_MAX = 64;
|
|
|
1011
1055
|
//
|
|
1012
1056
|
// The row's config holds only `activeVersionId` plus the egress/limit settings;
|
|
1013
1057
|
// names and schemas live on the version. That's what keeps MCP boot off the
|
|
1014
|
-
// dispatcher — a slow or broken script must never break tools/list.
|
|
1015
|
-
// docs/CUSTOM_TOOLS.md ("Boot contract").
|
|
1058
|
+
// dispatcher — a slow or broken script must never break tools/list.
|
|
1016
1059
|
const TOOL_DEFINITION_KEY_CUSTOM_CODE = 'custom-code';
|
|
1017
1060
|
// A version is uploaded as `draft`, becomes `published` when it is made active,
|
|
1018
1061
|
// and is `archived` when a later version replaces it. Exactly one version per
|
|
@@ -1692,7 +1735,8 @@ const PLAN_LIMITS = {
|
|
|
1692
1735
|
includedToolCalls: 0,
|
|
1693
1736
|
toolCallHardCap: 10_000,
|
|
1694
1737
|
includedMessages: 100,
|
|
1695
|
-
includedEmbeddedBytes: 5 * MB
|
|
1738
|
+
includedEmbeddedBytes: 5 * MB,
|
|
1739
|
+
autoSyncIntervals: []
|
|
1696
1740
|
},
|
|
1697
1741
|
PRO: {
|
|
1698
1742
|
maxProjects: null,
|
|
@@ -1715,7 +1759,11 @@ const PLAN_LIMITS = {
|
|
|
1715
1759
|
includedToolCalls: PRICING_INCLUDED_TOOL_CALLS,
|
|
1716
1760
|
toolCallHardCap: PRICING_TOOL_CALL_HARD_CAP,
|
|
1717
1761
|
includedMessages: PRICING_INCLUDED_MESSAGES,
|
|
1718
|
-
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
|
|
1762
|
+
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
|
|
1763
|
+
autoSyncIntervals: [
|
|
1764
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
1765
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
1766
|
+
]
|
|
1719
1767
|
},
|
|
1720
1768
|
ENTERPRISE: {
|
|
1721
1769
|
maxProjects: null,
|
|
@@ -1738,7 +1786,11 @@ const PLAN_LIMITS = {
|
|
|
1738
1786
|
// contract; don't make one plan's abuse limit another plan's product limit.
|
|
1739
1787
|
toolCallHardCap: null,
|
|
1740
1788
|
includedMessages: PRICING_INCLUDED_MESSAGES,
|
|
1741
|
-
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
|
|
1789
|
+
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
|
|
1790
|
+
autoSyncIntervals: [
|
|
1791
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
1792
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
1793
|
+
]
|
|
1742
1794
|
}
|
|
1743
1795
|
};
|
|
1744
1796
|
// Quota features — used as the `feature` discriminator on PlanLimitError so the
|
|
@@ -1756,6 +1808,7 @@ const PLAN_FEATURE_RAW_STORAGE = 'rawStorage';
|
|
|
1756
1808
|
const PLAN_FEATURE_EMBEDDED_STORAGE = 'embeddedStorage';
|
|
1757
1809
|
const PLAN_FEATURE_MESSAGE = 'message';
|
|
1758
1810
|
const PLAN_FEATURE_TOOL_CALL = 'toolCall';
|
|
1811
|
+
const PLAN_FEATURE_RESOURCE_SYNC = 'resourceSync';
|
|
1759
1812
|
// Stable code returned on a quota block so clients can branch on it (402).
|
|
1760
1813
|
const PLAN_LIMIT_ERROR_CODE = 'PLAN_LIMIT_EXCEEDED';
|
|
1761
1814
|
// Billing meter event names. The metering cron reports per-period OVERAGE
|
|
@@ -1923,6 +1976,7 @@ exports.constants = {
|
|
|
1923
1976
|
PLAN_FEATURE_EMBEDDED_STORAGE,
|
|
1924
1977
|
PLAN_FEATURE_MESSAGE,
|
|
1925
1978
|
PLAN_FEATURE_TOOL_CALL,
|
|
1979
|
+
PLAN_FEATURE_RESOURCE_SYNC,
|
|
1926
1980
|
PLAN_LIMIT_ERROR_CODE,
|
|
1927
1981
|
BILLING_METER_MESSAGES,
|
|
1928
1982
|
BILLING_METER_SHARED_MESSAGES,
|
|
@@ -2117,6 +2171,14 @@ exports.constants = {
|
|
|
2117
2171
|
RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER,
|
|
2118
2172
|
RESOURCE_SOURCE_TYPE_CUSTOM_CODE,
|
|
2119
2173
|
RESOURCE_SOURCE_TYPES,
|
|
2174
|
+
RESOURCE_SYNC_INTERVAL_OFF,
|
|
2175
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
2176
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY,
|
|
2177
|
+
RESOURCE_SYNC_INTERVALS,
|
|
2178
|
+
RESOURCE_SYNC_INTERVAL_DEFAULT,
|
|
2179
|
+
RESOURCE_SYNC_INTERVAL_MS,
|
|
2180
|
+
RESOURCE_SYNC_BATCH_SIZE,
|
|
2181
|
+
RESOURCE_SYNC_MANUAL_COOLDOWN_MS,
|
|
2120
2182
|
CRAWL_RENDERER_CHEERIO,
|
|
2121
2183
|
CRAWL_RENDERER_PLAYWRIGHT,
|
|
2122
2184
|
CRAWL_RENDERERS,
|
|
@@ -2238,6 +2300,12 @@ exports.constants = {
|
|
|
2238
2300
|
CHUNK_TARGET_CHARS,
|
|
2239
2301
|
CHUNK_OVERLAP_CHARS,
|
|
2240
2302
|
EMBED_BATCH_SIZE,
|
|
2303
|
+
RESOURCE_SEARCH_CANDIDATES,
|
|
2304
|
+
RESOURCE_SEARCH_RRF_K,
|
|
2305
|
+
RESOURCE_SEARCH_COMMON_TERM_DIVISOR,
|
|
2306
|
+
RESOURCE_SEARCH_COMMON_TERM_MIN,
|
|
2307
|
+
RESOURCE_SEARCH_RERANK_CANDIDATES,
|
|
2308
|
+
RESOURCE_SEARCH_RERANK_MODEL,
|
|
2241
2309
|
RESOURCE_HANDLER_SLEEP_AFTER,
|
|
2242
2310
|
DOCS_URL,
|
|
2243
2311
|
BASE64_DATA_URI_RE,
|