@ganju/utils 0.0.9 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channelDebounce.d.ts +11 -0
- package/dist/channelDebounce.d.ts.map +1 -1
- package/dist/channelDebounce.js +12 -1
- package/dist/chunking.d.ts +17 -3
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +125 -19
- package/dist/constants.d.ts +30 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +136 -5
- package/dist/index.d.ts +207 -28
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +16 -0
- package/dist/resourceRerank.d.ts +20 -0
- package/dist/resourceRerank.d.ts.map +1 -0
- package/dist/resourceRerank.js +37 -0
- package/dist/schema.d.ts +106 -24
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +45 -4
- package/dist/syncableResource.d.ts +14 -0
- package/dist/syncableResource.d.ts.map +1 -0
- package/dist/syncableResource.js +43 -0
- package/dist/toolCatalog.d.ts +70 -1
- package/dist/toolCatalog.d.ts.map +1 -1
- package/dist/toolCatalog.js +151 -63
- package/dist/toolConfirmation.d.ts +47 -0
- package/dist/toolConfirmation.d.ts.map +1 -0
- package/dist/toolConfirmation.js +217 -0
- package/package.json +1 -1
- package/src/channelDebounce.ts +36 -0
- package/src/chunking.ts +166 -23
- package/src/constants.ts +149 -5
- package/src/index.ts +51 -1
- package/src/resourceRerank.ts +54 -0
- package/src/schema.ts +58 -6
- package/src/syncableResource.ts +68 -0
- package/src/toolCatalog.ts +170 -63
- package/src/toolConfirmation.ts +261 -0
|
@@ -2,7 +2,17 @@ export interface BufferedChannelMessage {
|
|
|
2
2
|
text: string;
|
|
3
3
|
externalMessageId: string | null;
|
|
4
4
|
receivedAt: number;
|
|
5
|
+
immediate?: boolean;
|
|
6
|
+
command?: BufferedChannelCommand | null;
|
|
7
|
+
reply?: Record<string, unknown> | null;
|
|
5
8
|
}
|
|
9
|
+
export interface BufferedChannelCommand {
|
|
10
|
+
promptId: string;
|
|
11
|
+
artifactPromptId: string | null;
|
|
12
|
+
promptTitle: string;
|
|
13
|
+
args: Record<string, string>;
|
|
14
|
+
}
|
|
15
|
+
export declare const batchCommand: (messages: BufferedChannelMessage[]) => BufferedChannelCommand | null;
|
|
6
16
|
export interface ChannelBufferEnvelope {
|
|
7
17
|
channelId: string;
|
|
8
18
|
platform: string;
|
|
@@ -17,6 +27,7 @@ export interface ChannelBufferEnvelope {
|
|
|
17
27
|
export interface ChannelBufferFlush {
|
|
18
28
|
envelope: ChannelBufferEnvelope;
|
|
19
29
|
messages: BufferedChannelMessage[];
|
|
30
|
+
batchId: string;
|
|
20
31
|
}
|
|
21
32
|
export declare const resolveDebounceMs: (config: unknown) => number;
|
|
22
33
|
export declare const channelBufferKey: (channelId: string, externalConversationId: string, externalParticipantId: string) => string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"channelDebounce.d.ts","sourceRoot":"","sources":["../src/channelDebounce.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"channelDebounce.d.ts","sourceRoot":"","sources":["../src/channelDebounce.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IAKnB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,OAAO,CAAC,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAKxC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACxC;AAID,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAID,eAAO,MAAM,YAAY,GACvB,UAAU,sBAAsB,EAAE,KACjC,sBAAsB,GAAG,IAM3B,CAAC;AAOF,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACpD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAGD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,qBAAqB,CAAC;IAChC,QAAQ,EAAE,sBAAsB,EAAE,CAAC;IAGnC,OAAO,EAAE,MAAM,CAAC;CACjB;AAOD,eAAO,MAAM,iBAAiB,GAAI,QAAQ,OAAO,KAAG,MAYnD,CAAC;AAKF,eAAO,MAAM,gBAAgB,GAC3B,WAAW,MAAM,EACjB,wBAAwB,MAAM,EAC9B,uBAAuB,MAAM,KAC5B,MAA2E,CAAC;AAI/E,eAAO,MAAM,oBAAoB,GAC/B,UAAU,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,KAChC,MAIuC,CAAC"}
|
package/dist/channelDebounce.js
CHANGED
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.joinBufferedMessages = exports.channelBufferKey = exports.resolveDebounceMs = void 0;
|
|
3
|
+
exports.joinBufferedMessages = exports.channelBufferKey = exports.resolveDebounceMs = exports.batchCommand = void 0;
|
|
4
4
|
const constants_1 = require("./constants");
|
|
5
|
+
// The prompt a batch runs: the newest command in it, since that is the one the
|
|
6
|
+
// participant asked for last. Null for a batch of plain text.
|
|
7
|
+
const batchCommand = (messages) => {
|
|
8
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
9
|
+
const command = messages[i].command;
|
|
10
|
+
if (command)
|
|
11
|
+
return command;
|
|
12
|
+
}
|
|
13
|
+
return null;
|
|
14
|
+
};
|
|
15
|
+
exports.batchCommand = batchCommand;
|
|
5
16
|
// Read the channel's debounce window off its `config`, clamped to the supported
|
|
6
17
|
// range. An unset value takes the default; an explicit 0 disables buffering, so
|
|
7
18
|
// the channel answers every message as it arrives. Anything malformed (a string,
|
package/dist/chunking.d.ts
CHANGED
|
@@ -6,6 +6,17 @@ export declare const chunkText: (text: string, options?: {
|
|
|
6
6
|
targetChars?: number;
|
|
7
7
|
overlapChars?: number;
|
|
8
8
|
}) => string[];
|
|
9
|
+
export type HeadingSyntax = 'markdown' | 'html';
|
|
10
|
+
/**
|
|
11
|
+
* Which heading syntax a document is written in, or null when it has none we
|
|
12
|
+
* can trust. Plain text is left out on purpose: a line that happens to start
|
|
13
|
+
* with "# " there is as likely a comment or a list as a title.
|
|
14
|
+
*/
|
|
15
|
+
export declare const headingSyntaxFor: (input: {
|
|
16
|
+
mimeType?: string | null;
|
|
17
|
+
fileName?: string | null;
|
|
18
|
+
isWebPage?: boolean;
|
|
19
|
+
}) => HeadingSyntax | null;
|
|
9
20
|
export declare const buildHeader: (input: {
|
|
10
21
|
title?: string | null;
|
|
11
22
|
description?: string | null;
|
|
@@ -13,9 +24,10 @@ export declare const buildHeader: (input: {
|
|
|
13
24
|
mimeType?: string | null;
|
|
14
25
|
fileName?: string | null;
|
|
15
26
|
}) => string;
|
|
16
|
-
export interface ChunkMetadata extends ExtractedDocumentMetadata {
|
|
17
|
-
chunkIndexInPage
|
|
18
|
-
chunksInPage
|
|
27
|
+
export interface ChunkMetadata extends Partial<ExtractedDocumentMetadata> {
|
|
28
|
+
chunkIndexInPage?: number;
|
|
29
|
+
chunksInPage?: number;
|
|
30
|
+
headingPath?: string[];
|
|
19
31
|
}
|
|
20
32
|
export interface PreparedChunk {
|
|
21
33
|
content: string;
|
|
@@ -24,5 +36,7 @@ export interface PreparedChunk {
|
|
|
24
36
|
export declare const prepareChunks: (header: string, documents: ExtractedDocument[] | null, fallbackContent: string | null, options?: {
|
|
25
37
|
targetChars?: number;
|
|
26
38
|
overlapChars?: number;
|
|
39
|
+
mimeType?: string | null;
|
|
40
|
+
fileName?: string | null;
|
|
27
41
|
}) => PreparedChunk[];
|
|
28
42
|
//# sourceMappingURL=chunking.d.ts.map
|
package/dist/chunking.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,KAAK,EACV,iBAAiB,EACjB,yBAAyB,EAC1B,MAAM,qBAAqB,CAAC;AAE7B,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,SAAS,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AASpE,eAAO,MAAM,cAAc,GACzB,MAAM,MAAM,EACZ,QAAQ,MAAM,EACd,aAAY,SAAS,SAAS,EAA+B,KAC5D,MAAM,EA8BR,CAAC;AA4BF,eAAO,MAAM,SAAS,GACpB,MAAM,MAAM,EACZ,UAAU;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KACxD,MAAM,EAAoD,CAAC;AAE9D,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,MAAM,CAAC;AAehD;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,GAAI,OAAO;IACtC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,KAAG,aAAa,GAAG,IAUnB,CAAC;AAuEF,eAAO,MAAM,WAAW,GAAI,OAAO;IACjC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KAAG,MASW,CAAC;AAIhB,MAAM,WAAW,aAAc,SAAQ,OAAO,CAAC,yBAAyB,CAAC;IACvE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CAChC;AAID,eAAO,MAAM,aAAa,GACxB,QAAQ,MAAM,EACd,WAAW,iBAAiB,EAAE,GAAG,IAAI,EACrC,iBAAiB,MAAM,GAAG,IAAI,EAC9B,UAAU;IACR,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KACA,aAAa,EA4Cf,CAAC"}
|
package/dist/chunking.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.prepareChunks = exports.buildHeader = exports.chunkText = exports.splitRecursive = void 0;
|
|
3
|
+
exports.prepareChunks = exports.buildHeader = exports.headingSyntaxFor = exports.chunkText = exports.splitRecursive = void 0;
|
|
4
4
|
const constants_1 = require("./constants");
|
|
5
|
+
const sanitize_1 = require("./sanitize");
|
|
5
6
|
const trySplit = (text, sep) => {
|
|
6
7
|
if (typeof sep === 'string') {
|
|
7
8
|
return text.includes(sep) ? text.split(sep) : null;
|
|
@@ -40,23 +41,115 @@ const splitRecursive = (text, target, separators = constants_1.constants.CHUNK_S
|
|
|
40
41
|
return chunks;
|
|
41
42
|
};
|
|
42
43
|
exports.splitRecursive = splitRecursive;
|
|
43
|
-
|
|
44
|
+
// Each piece keeps its `raw` text — the slice of the input it covers, without
|
|
45
|
+
// the overlap borrowed from the piece before — so a caller can walk the input
|
|
46
|
+
// in order without reading any of it twice.
|
|
47
|
+
const chunkPieces = (text, options) => {
|
|
44
48
|
const target = options?.targetChars ?? constants_1.constants.CHUNK_TARGET_CHARS;
|
|
45
49
|
const overlap = options?.overlapChars ?? constants_1.constants.CHUNK_OVERLAP_CHARS;
|
|
46
50
|
const trimmed = text.trim();
|
|
47
51
|
if (!trimmed)
|
|
48
52
|
return [];
|
|
49
53
|
if (trimmed.length <= target)
|
|
50
|
-
return [trimmed];
|
|
54
|
+
return [{ raw: trimmed, content: trimmed }];
|
|
51
55
|
const rawChunks = (0, exports.splitRecursive)(trimmed, target);
|
|
52
|
-
const
|
|
56
|
+
const pieces = [];
|
|
53
57
|
for (let i = 0; i < rawChunks.length; i++) {
|
|
54
58
|
const prev = i > 0 ? rawChunks[i - 1].slice(-overlap) : '';
|
|
55
|
-
|
|
59
|
+
pieces.push({
|
|
60
|
+
raw: rawChunks[i],
|
|
61
|
+
content: (prev ? prev + '\n' : '') + rawChunks[i]
|
|
62
|
+
});
|
|
56
63
|
}
|
|
57
|
-
return
|
|
64
|
+
return pieces.filter(p => p.content.trim().length > 0);
|
|
58
65
|
};
|
|
66
|
+
const chunkText = (text, options) => chunkPieces(text, options).map(p => p.content);
|
|
59
67
|
exports.chunkText = chunkText;
|
|
68
|
+
const MARKDOWN_HEADING = /^ {0,3}(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/;
|
|
69
|
+
const MARKDOWN_FENCE = /^ {0,3}(```|~~~)/;
|
|
70
|
+
const HTML_HEADING = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi;
|
|
71
|
+
const MAX_HEADING_CHARS = 120;
|
|
72
|
+
const MARKDOWN_FILE_NAME = /\.(md|markdown|mdx)$/i;
|
|
73
|
+
/**
|
|
74
|
+
* Which heading syntax a document is written in, or null when it has none we
|
|
75
|
+
* can trust. Plain text is left out on purpose: a line that happens to start
|
|
76
|
+
* with "# " there is as likely a comment or a list as a title.
|
|
77
|
+
*/
|
|
78
|
+
const headingSyntaxFor = (input) => {
|
|
79
|
+
// Crawled pages are flattened to text with their headings written back as
|
|
80
|
+
// Markdown lines, whatever the page's own content type was.
|
|
81
|
+
if (input.isWebPage)
|
|
82
|
+
return 'markdown';
|
|
83
|
+
if (input.mimeType === constants_1.constants.MIMETYPE_TEXT_MARKDOWN)
|
|
84
|
+
return 'markdown';
|
|
85
|
+
if (input.mimeType === constants_1.constants.MIMETYPE_TEXT_HTML)
|
|
86
|
+
return 'html';
|
|
87
|
+
if (input.fileName && MARKDOWN_FILE_NAME.test(input.fileName)) {
|
|
88
|
+
return 'markdown';
|
|
89
|
+
}
|
|
90
|
+
return null;
|
|
91
|
+
};
|
|
92
|
+
exports.headingSyntaxFor = headingSyntaxFor;
|
|
93
|
+
const cleanHeading = (text) => text.replace(/\s+/g, ' ').trim().slice(0, MAX_HEADING_CHARS);
|
|
94
|
+
const scanMarkdownHeadings = (text, state) => {
|
|
95
|
+
const events = [];
|
|
96
|
+
let offset = 0;
|
|
97
|
+
for (const line of text.split('\n')) {
|
|
98
|
+
// A `#` inside a fenced block is a shell comment or a preprocessor line,
|
|
99
|
+
// not a title — and a fence can open in one chunk and close in the next.
|
|
100
|
+
if (MARKDOWN_FENCE.test(line)) {
|
|
101
|
+
state.inFence = !state.inFence;
|
|
102
|
+
}
|
|
103
|
+
else if (!state.inFence) {
|
|
104
|
+
const match = MARKDOWN_HEADING.exec(line);
|
|
105
|
+
const heading = match ? cleanHeading(match[2]) : '';
|
|
106
|
+
if (match && heading) {
|
|
107
|
+
events.push({ offset, level: match[1].length, text: heading });
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
offset += line.length + 1;
|
|
111
|
+
}
|
|
112
|
+
return events;
|
|
113
|
+
};
|
|
114
|
+
const scanHtmlHeadings = (text) => [...text.matchAll(HTML_HEADING)]
|
|
115
|
+
.map(match => ({
|
|
116
|
+
offset: match.index ?? 0,
|
|
117
|
+
level: Number(match[1]),
|
|
118
|
+
text: cleanHeading((0, sanitize_1.decodeEntities)(match[2].replace(/<[^>]*>/g, ' ')))
|
|
119
|
+
}))
|
|
120
|
+
.filter(event => event.text);
|
|
121
|
+
/**
|
|
122
|
+
* Follows the heading hierarchy across a document's chunks, fed in order.
|
|
123
|
+
* Returns, for each chunk, the path of headings it sits under — the section a
|
|
124
|
+
* reader would be pointed to. A chunk that opens with a heading belongs to that
|
|
125
|
+
* heading; one that starts mid-section belongs to the section it continues,
|
|
126
|
+
* even when a later heading begins inside it.
|
|
127
|
+
*/
|
|
128
|
+
const createHeadingTracker = (syntax) => {
|
|
129
|
+
let stack = [];
|
|
130
|
+
const fence = { inFence: false };
|
|
131
|
+
const apply = (event) => {
|
|
132
|
+
stack = stack.filter(h => h.level < event.level);
|
|
133
|
+
stack.push({ level: event.level, text: event.text });
|
|
134
|
+
};
|
|
135
|
+
return (raw) => {
|
|
136
|
+
const events = syntax === 'markdown'
|
|
137
|
+
? scanMarkdownHeadings(raw, fence)
|
|
138
|
+
: scanHtmlHeadings(raw);
|
|
139
|
+
const [first, ...rest] = events;
|
|
140
|
+
// Before a document's first heading there is no section to continue — only
|
|
141
|
+
// the metadata header, or the markup around an HTML body — so a chunk that
|
|
142
|
+
// holds the first heading belongs to it wherever in the chunk it falls.
|
|
143
|
+
const opensWithHeading = first !== undefined &&
|
|
144
|
+
(stack.length === 0 || raw.slice(0, first.offset).trim() === '');
|
|
145
|
+
if (opensWithHeading)
|
|
146
|
+
apply(first);
|
|
147
|
+
const path = stack.map(h => h.text);
|
|
148
|
+
for (const event of opensWithHeading ? rest : events)
|
|
149
|
+
apply(event);
|
|
150
|
+
return path;
|
|
151
|
+
};
|
|
152
|
+
};
|
|
60
153
|
const buildHeader = (input) => [
|
|
61
154
|
input.title ? `Title: ${input.title}` : null,
|
|
62
155
|
input.description ? `Description: ${input.description}` : null,
|
|
@@ -67,37 +160,50 @@ const buildHeader = (input) => [
|
|
|
67
160
|
.filter(Boolean)
|
|
68
161
|
.join('\n');
|
|
69
162
|
exports.buildHeader = buildHeader;
|
|
163
|
+
const CONTROL_CHARS = /[\x00-\x08\x0B\x0C\x0E-\x1F]/g;
|
|
70
164
|
const prepareChunks = (header, documents, fallbackContent, options) => {
|
|
71
165
|
if (documents && documents.length > 0) {
|
|
72
166
|
const out = [];
|
|
73
167
|
documents.forEach((doc, docIndex) => {
|
|
74
|
-
const body = (doc.pageContent || '')
|
|
75
|
-
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
|
|
76
|
-
.trim();
|
|
168
|
+
const body = (doc.pageContent || '').replace(CONTROL_CHARS, '').trim();
|
|
77
169
|
if (!body)
|
|
78
170
|
return;
|
|
79
171
|
const text = docIndex === 0 && header ? `${header}\n\n${body}` : body;
|
|
80
|
-
const
|
|
172
|
+
const syntax = (0, exports.headingSyntaxFor)({
|
|
173
|
+
mimeType: doc.metadata?.source?.mimeType ?? options?.mimeType,
|
|
174
|
+
fileName: doc.metadata?.source?.fileName ?? options?.fileName,
|
|
175
|
+
isWebPage: !!doc.metadata?.web
|
|
176
|
+
});
|
|
177
|
+
const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
|
|
178
|
+
const pieces = chunkPieces(text, options);
|
|
81
179
|
pieces.forEach((piece, pieceIndex) => {
|
|
180
|
+
const headingPath = headingPathOf?.(piece.raw) ?? [];
|
|
82
181
|
out.push({
|
|
83
|
-
content: piece,
|
|
182
|
+
content: piece.content,
|
|
84
183
|
metadata: {
|
|
85
184
|
...doc.metadata,
|
|
86
185
|
chunkIndexInPage: pieceIndex,
|
|
87
|
-
chunksInPage: pieces.length
|
|
186
|
+
chunksInPage: pieces.length,
|
|
187
|
+
...(headingPath.length > 0 ? { headingPath } : {})
|
|
88
188
|
}
|
|
89
189
|
});
|
|
90
190
|
});
|
|
91
191
|
});
|
|
92
192
|
return out;
|
|
93
193
|
}
|
|
94
|
-
const body = (fallbackContent || '')
|
|
95
|
-
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
|
|
96
|
-
.trim();
|
|
194
|
+
const body = (fallbackContent || '').replace(CONTROL_CHARS, '').trim();
|
|
97
195
|
const fullText = [header, body].filter(Boolean).join('\n\n');
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
})
|
|
196
|
+
const syntax = (0, exports.headingSyntaxFor)({
|
|
197
|
+
mimeType: options?.mimeType,
|
|
198
|
+
fileName: options?.fileName
|
|
199
|
+
});
|
|
200
|
+
const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
|
|
201
|
+
return chunkPieces(fullText, options).map(piece => {
|
|
202
|
+
const headingPath = headingPathOf?.(piece.raw) ?? [];
|
|
203
|
+
return {
|
|
204
|
+
content: piece.content,
|
|
205
|
+
metadata: headingPath.length > 0 ? { headingPath } : null
|
|
206
|
+
};
|
|
207
|
+
});
|
|
102
208
|
};
|
|
103
209
|
exports.prepareChunks = prepareChunks;
|
package/dist/constants.d.ts
CHANGED
|
@@ -6,6 +6,8 @@ declare const ONE_DRIVE_TAB_MY_FILES: "my-files";
|
|
|
6
6
|
declare const ONE_DRIVE_TAB_SHARED_WITH_ME: "shared-with-me";
|
|
7
7
|
declare const ONE_DRIVE_TAB_RECENT: "recent";
|
|
8
8
|
declare const ONE_DRIVE_TAB_DRIVES: "drives";
|
|
9
|
+
declare const RESOURCE_SYNC_INTERVAL_DAILY: "daily";
|
|
10
|
+
declare const RESOURCE_SYNC_INTERVAL_WEEKLY: "weekly";
|
|
9
11
|
declare const LLM_PROVIDER_ANTHROPIC: "anthropic";
|
|
10
12
|
declare const LLM_PROVIDER_OPENAI: "openai";
|
|
11
13
|
declare const LLM_PROVIDER_GOOGLE: "google";
|
|
@@ -59,6 +61,7 @@ interface PlanLimits {
|
|
|
59
61
|
toolCallHardCap: number | null;
|
|
60
62
|
includedMessages: number;
|
|
61
63
|
includedEmbeddedBytes: number;
|
|
64
|
+
autoSyncIntervals: ReadonlyArray<typeof RESOURCE_SYNC_INTERVAL_DAILY | typeof RESOURCE_SYNC_INTERVAL_WEEKLY>;
|
|
62
65
|
}
|
|
63
66
|
export type { PlanLimits };
|
|
64
67
|
export declare const constants: {
|
|
@@ -104,6 +107,7 @@ export declare const constants: {
|
|
|
104
107
|
PLAN_FEATURE_EMBEDDED_STORAGE: "embeddedStorage";
|
|
105
108
|
PLAN_FEATURE_MESSAGE: "message";
|
|
106
109
|
PLAN_FEATURE_TOOL_CALL: "toolCall";
|
|
110
|
+
PLAN_FEATURE_RESOURCE_SYNC: "resourceSync";
|
|
107
111
|
PLAN_LIMIT_ERROR_CODE: string;
|
|
108
112
|
BILLING_METER_MESSAGES: string;
|
|
109
113
|
BILLING_METER_SHARED_MESSAGES: string;
|
|
@@ -309,6 +313,18 @@ export declare const constants: {
|
|
|
309
313
|
RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER: "ONE_DRIVE_FOLDER";
|
|
310
314
|
RESOURCE_SOURCE_TYPE_CUSTOM_CODE: "CUSTOM_CODE";
|
|
311
315
|
RESOURCE_SOURCE_TYPES: ("FILE" | "WEBSITE" | "GOOGLE_DRIVE_FOLDER" | "ONE_DRIVE_FOLDER" | "CUSTOM_CODE")[];
|
|
316
|
+
RESOURCE_SYNC_INTERVAL_OFF: "off";
|
|
317
|
+
RESOURCE_SYNC_INTERVAL_DAILY: "daily";
|
|
318
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY: "weekly";
|
|
319
|
+
RESOURCE_SYNC_INTERVALS: readonly ["off", "daily", "weekly"];
|
|
320
|
+
RESOURCE_SYNC_INTERVAL_DEFAULT: "weekly";
|
|
321
|
+
RESOURCE_SYNC_INTERVAL_MS: {
|
|
322
|
+
readonly daily: number;
|
|
323
|
+
readonly weekly: number;
|
|
324
|
+
};
|
|
325
|
+
RESOURCE_SYNC_BATCH_SIZE: number;
|
|
326
|
+
RESOURCE_SYNC_MANUAL_COOLDOWN_MS: number;
|
|
327
|
+
RESOURCE_SYNC_STALE_MS: number;
|
|
312
328
|
CRAWL_RENDERER_CHEERIO: "cheerio";
|
|
313
329
|
CRAWL_RENDERER_PLAYWRIGHT: "playwright";
|
|
314
330
|
CRAWL_RENDERERS: ("cheerio" | "playwright")[];
|
|
@@ -371,6 +387,11 @@ export declare const constants: {
|
|
|
371
387
|
SHARED_KEY_HISTORY_LIMIT: number;
|
|
372
388
|
SHARED_KEY_MAX_TOOL_LOOPS: number;
|
|
373
389
|
CHANNEL_MAX_TOOLS: number;
|
|
390
|
+
TOOL_CONFIRMATION_TTL_MS: number;
|
|
391
|
+
TOOL_CONFIRMATION_REQUIRED_RESULT: string;
|
|
392
|
+
TOOL_CONFIRMATION_FALLBACK_QUESTION: string;
|
|
393
|
+
TOOL_CONFIRMATION_ASK_INSTRUCTION: string;
|
|
394
|
+
TOOL_CONFIRMATION_SYSTEM_NOTE: string;
|
|
374
395
|
CHANNEL_DEBOUNCE_DEFAULT_MS: number;
|
|
375
396
|
CHANNEL_DEBOUNCE_MIN_MS: number;
|
|
376
397
|
CHANNEL_DEBOUNCE_MAX_MS: number;
|
|
@@ -403,8 +424,10 @@ export declare const constants: {
|
|
|
403
424
|
DISCORD_TIMESTAMP_HEADER: string;
|
|
404
425
|
DISCORD_INTERACTION_TYPE_PING: number;
|
|
405
426
|
DISCORD_INTERACTION_TYPE_APPLICATION_COMMAND: number;
|
|
427
|
+
DISCORD_INTERACTION_TYPE_MESSAGE_COMPONENT: number;
|
|
406
428
|
DISCORD_INTERACTION_RESPONSE_PONG: number;
|
|
407
429
|
DISCORD_INTERACTION_RESPONSE_DEFERRED: number;
|
|
430
|
+
DISCORD_INTERACTION_RESPONSE_UPDATE_MESSAGE: number;
|
|
408
431
|
DISCORD_CHANNEL_TYPE_DM: number;
|
|
409
432
|
DISCORD_CHANNEL_TYPE_GROUP_DM: number;
|
|
410
433
|
DISCORD_GATEWAY_OP_DISPATCH: number;
|
|
@@ -419,6 +442,7 @@ export declare const constants: {
|
|
|
419
442
|
WHATSAPP_API_BASE: string;
|
|
420
443
|
WHATSAPP_API_VERSION: string;
|
|
421
444
|
WHATSAPP_MESSAGE_LIMIT: number;
|
|
445
|
+
WHATSAPP_INTERACTIVE_BODY_LIMIT: number;
|
|
422
446
|
WHATSAPP_SIGNATURE_HEADER: string;
|
|
423
447
|
WHATSAPP_SIGNATURE_PREFIX: string;
|
|
424
448
|
WHATSAPP_HUB_MODE_PARAM: string;
|
|
@@ -434,6 +458,12 @@ export declare const constants: {
|
|
|
434
458
|
CHUNK_TARGET_CHARS: number;
|
|
435
459
|
CHUNK_OVERLAP_CHARS: number;
|
|
436
460
|
EMBED_BATCH_SIZE: number;
|
|
461
|
+
RESOURCE_SEARCH_CANDIDATES: number;
|
|
462
|
+
RESOURCE_SEARCH_RRF_K: number;
|
|
463
|
+
RESOURCE_SEARCH_COMMON_TERM_DIVISOR: number;
|
|
464
|
+
RESOURCE_SEARCH_COMMON_TERM_MIN: number;
|
|
465
|
+
RESOURCE_SEARCH_RERANK_CANDIDATES: number;
|
|
466
|
+
RESOURCE_SEARCH_RERANK_MODEL: string;
|
|
437
467
|
RESOURCE_HANDLER_SLEEP_AFTER: string;
|
|
438
468
|
DOCS_URL: string;
|
|
439
469
|
BASE64_DATA_URI_RE: RegExp;
|
package/dist/constants.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AA6JA,QAAA,MAAM,yBAAyB,EAAiB,UAAU,CAAC;AAC3D,QAAA,MAAM,+BAA+B,EAAuB,gBAAgB,CAAC;AAC7E,QAAA,MAAM,8BAA8B,EAAsB,eAAe,CAAC;AAC1E,QAAA,MAAM,wBAAwB,EAAgB,SAAS,CAAC;AAuCxD,QAAA,MAAM,sBAAsB,EAAiB,UAAU,CAAC;AACxD,QAAA,MAAM,4BAA4B,EAAuB,gBAAgB,CAAC;AAC1E,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAClD,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AA6JA,QAAA,MAAM,yBAAyB,EAAiB,UAAU,CAAC;AAC3D,QAAA,MAAM,+BAA+B,EAAuB,gBAAgB,CAAC;AAC7E,QAAA,MAAM,8BAA8B,EAAsB,eAAe,CAAC;AAC1E,QAAA,MAAM,wBAAwB,EAAgB,SAAS,CAAC;AAuCxD,QAAA,MAAM,sBAAsB,EAAiB,UAAU,CAAC;AACxD,QAAA,MAAM,4BAA4B,EAAuB,gBAAgB,CAAC;AAC1E,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAClD,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAoFlD,QAAA,MAAM,4BAA4B,EAAc,OAAO,CAAC;AACxD,QAAA,MAAM,6BAA6B,EAAe,QAAQ,CAAC;AAmV3D,QAAA,MAAM,sBAAsB,EAAkB,WAAW,CAAC;AAC1D,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAGjD,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAyYjD,MAAM,MAAM,mBAAmB,GAC3B;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC9D;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA0+BpE,UAAU,UAAU;IAElB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAGhC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAMjC,sBAAsB,EAAE,MAAM,CAAC;IAM/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,SAAS,EAAE,OAAO,CAAC;IAKnB,eAAe,EAAE,OAAO,CAAC;IAOzB,gBAAgB,EAAE,OAAO,CAAC;IAK1B,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAQ3C,iBAAiB,EAAE,MAAM,CAAC;IAM1B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAG/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAI9B,iBAAiB,EAAE,aAAa,CAC9B,OAAO,4BAA4B,GAAG,OAAO,6BAA6B,CAC3E,CAAC;CACH;AA2PD,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eA5oEhB,OAAO,yBAAyB,GAChC,OAAO,+BAA+B,GACtC,OAAO,8BAA8B,GACrC,OAAO,wBAAwB;eAC5B,MAAM;;;;;;;;;;;;;;;;;;eAsCT,OAAO,sBAAsB,GAC7B,OAAO,4BAA4B,GACnC,OAAO,oBAAoB,GAC3B,OAAO,oBAAoB;eACxB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkbT,OAAO,mBAAmB,GAC1B,OAAO,mBAAmB,GAC1B,OAAO,sBAAsB;eAC1B,MAAM;eACN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAoTwB,MAAM,GAAG,MAAM;cAAQ,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAm9DnE,CAAC"}
|
package/dist/constants.js
CHANGED
|
@@ -208,6 +208,42 @@ const RESOURCE_SOURCE_TYPES = [
|
|
|
208
208
|
RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER,
|
|
209
209
|
RESOURCE_SOURCE_TYPE_CUSTOM_CODE
|
|
210
210
|
];
|
|
211
|
+
// How often a source that lives somewhere else — a crawled website, a Drive or
|
|
212
|
+
// OneDrive folder, a file imported from either — is refreshed from it. Every
|
|
213
|
+
// source is stored with an interval, weekly unless its owner changes it; the
|
|
214
|
+
// hourly sync job only acts on it for a plan that includes automatic sync, so a
|
|
215
|
+
// Free org's sources start syncing the moment it upgrades.
|
|
216
|
+
const RESOURCE_SYNC_INTERVAL_OFF = 'off';
|
|
217
|
+
const RESOURCE_SYNC_INTERVAL_DAILY = 'daily';
|
|
218
|
+
const RESOURCE_SYNC_INTERVAL_WEEKLY = 'weekly';
|
|
219
|
+
const RESOURCE_SYNC_INTERVALS = [
|
|
220
|
+
RESOURCE_SYNC_INTERVAL_OFF,
|
|
221
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
222
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
223
|
+
];
|
|
224
|
+
const RESOURCE_SYNC_INTERVAL_DEFAULT = RESOURCE_SYNC_INTERVAL_WEEKLY;
|
|
225
|
+
const RESOURCE_SYNC_INTERVAL_MS = {
|
|
226
|
+
[RESOURCE_SYNC_INTERVAL_DAILY]: 24 * 60 * 60 * 1000,
|
|
227
|
+
[RESOURCE_SYNC_INTERVAL_WEEKLY]: 7 * 24 * 60 * 60 * 1000
|
|
228
|
+
};
|
|
229
|
+
// Sources started per hourly run. A sync is cheap to start and expensive to
|
|
230
|
+
// run — a website re-crawls every page — so the job works through a backlog
|
|
231
|
+
// over several hours rather than launching all of it at once.
|
|
232
|
+
const RESOURCE_SYNC_BATCH_SIZE = 50;
|
|
233
|
+
// How soon a source can be synced by hand again. Manual sync is open to every
|
|
234
|
+
// plan, and a website sync re-crawls every page, so it can't be a button to
|
|
235
|
+
// press in a loop.
|
|
236
|
+
const RESOURCE_SYNC_MANUAL_COOLDOWN_MS = 5 * 60 * 1000;
|
|
237
|
+
// How long a sync may stay PENDING before it counts as stuck rather than
|
|
238
|
+
// running. A lost queue message or a job that exhausted its retries never
|
|
239
|
+
// completes its root, and a root that stays PENDING would otherwise be skipped
|
|
240
|
+
// by the hourly job and refused by "Sync now" forever. Twelve hours outlasts
|
|
241
|
+
// any realistic wait: the crawl-page queue runs 5 at a time for every org, and
|
|
242
|
+
// one hourly run can queue 50 sites — thousands of pages, hours of backlog. A
|
|
243
|
+
// shorter window restarts syncs that are merely queued, re-fetching every page
|
|
244
|
+
// into the same backlog; a longer one leaves a stuck daily source missing more
|
|
245
|
+
// than one sync.
|
|
246
|
+
const RESOURCE_SYNC_STALE_MS = 12 * 60 * 60 * 1000;
|
|
211
247
|
const CRAWL_RENDERER_CHEERIO = 'cheerio';
|
|
212
248
|
const CRAWL_RENDERER_PLAYWRIGHT = 'playwright';
|
|
213
249
|
const CRAWL_RENDERERS = [CRAWL_RENDERER_CHEERIO, CRAWL_RENDERER_PLAYWRIGHT];
|
|
@@ -594,6 +630,44 @@ const CHANNEL_DEBOUNCE_JOIN = '\n';
|
|
|
594
630
|
// and how many times it tries before dropping the batch rather than looping.
|
|
595
631
|
const CHANNEL_DEBOUNCE_RETRY_MS = 5000;
|
|
596
632
|
const CHANNEL_DEBOUNCE_MAX_ATTEMPTS = 3;
|
|
633
|
+
// Confirming sensitive actions, for organizations that turn it on. A channel
|
|
634
|
+
// bot that wants to run a tool annotated `destructiveHint` asks first, and the
|
|
635
|
+
// call runs only if the participant's next message is a plain yes.
|
|
636
|
+
//
|
|
637
|
+
// How long the question stays open. An answer after that is read as a new
|
|
638
|
+
// message, never as consent to something asked long ago.
|
|
639
|
+
const TOOL_CONFIRMATION_TTL_MS = 30 * 60 * 1000;
|
|
640
|
+
// The tool result the model gets instead of running the call. It is what makes
|
|
641
|
+
// the model write the question — in the participant's language, naming what
|
|
642
|
+
// the call would do — rather than claiming the action happened.
|
|
643
|
+
// The call's exact arguments are listed under the model's question by the
|
|
644
|
+
// runner, so the model only has to say what it is about to do.
|
|
645
|
+
const TOOL_CONFIRMATION_REQUIRED_RESULT = 'Not run yet: this action needs the user to confirm it first. In one or ' +
|
|
646
|
+
'two sentences, tell them what it will do (who it goes to, what will be ' +
|
|
647
|
+
'changed or deleted) and ask them to answer yes to go ahead or no to ' +
|
|
648
|
+
'cancel — in their own language, never quoting English words at someone ' +
|
|
649
|
+
'writing in another one. The exact details are listed for them under your ' +
|
|
650
|
+
'message. It runs only if their next message is yes. Do not say it was done.';
|
|
651
|
+
// Told to the model on every turn of an organization that confirms: the
|
|
652
|
+
// asking is done for it, so it calls the tool when asked instead of asking
|
|
653
|
+
// first itself — and calls every action a request needs in one step, since
|
|
654
|
+
// the turn stops at the first held call and each later one would be a
|
|
655
|
+
// question of its own.
|
|
656
|
+
const TOOL_CONFIRMATION_SYSTEM_NOTE = 'Actions that send, change or delete something are confirmed with the ' +
|
|
657
|
+
'user by the system before they run. Ask for any detail you are missing, ' +
|
|
658
|
+
'but once you have what the action needs, call the tool — do not ask the ' +
|
|
659
|
+
'user to confirm it yourself; the system asks them and runs it on their ' +
|
|
660
|
+
'yes. When a request needs several such actions (say, delete an event ' +
|
|
661
|
+
'and create its replacement), call them all in the same step, so the ' +
|
|
662
|
+
'user confirms them together once.';
|
|
663
|
+
// The system-prompt addition for the step that writes the question, with the
|
|
664
|
+
// held actions' names in place of {{actions}}.
|
|
665
|
+
const TOOL_CONFIRMATION_ASK_INSTRUCTION = 'IMPORTANT: these actions have NOT been done: {{actions}}. They are ' +
|
|
666
|
+
'waiting for the user to confirm them. Your reply must ask the user to ' +
|
|
667
|
+
'confirm, and must not say or suggest that they were done, nor describe ' +
|
|
668
|
+
'or make up any result from them.';
|
|
669
|
+
// The question when the model wrote none, above the runner's own summary.
|
|
670
|
+
const TOOL_CONFIRMATION_FALLBACK_QUESTION = 'Before I do this, please confirm. Reply yes to go ahead, or no to cancel.';
|
|
597
671
|
const TELEGRAM_SECRET_HEADER = 'x-telegram-bot-api-secret-token';
|
|
598
672
|
const TELEGRAM_API_BASE = 'https://api.telegram.org';
|
|
599
673
|
const TELEGRAM_MESSAGE_LIMIT = 3500;
|
|
@@ -650,8 +724,12 @@ const DISCORD_TIMESTAMP_HEADER = 'x-signature-timestamp';
|
|
|
650
724
|
// Interaction request types and response types (Discord API).
|
|
651
725
|
const DISCORD_INTERACTION_TYPE_PING = 1;
|
|
652
726
|
const DISCORD_INTERACTION_TYPE_APPLICATION_COMMAND = 2;
|
|
727
|
+
// A tap on a message button — the Yes/No under a confirmation question.
|
|
728
|
+
const DISCORD_INTERACTION_TYPE_MESSAGE_COMPONENT = 3;
|
|
653
729
|
const DISCORD_INTERACTION_RESPONSE_PONG = 1;
|
|
654
730
|
const DISCORD_INTERACTION_RESPONSE_DEFERRED = 5;
|
|
731
|
+
// Edits the message the tapped button is on, in the same response.
|
|
732
|
+
const DISCORD_INTERACTION_RESPONSE_UPDATE_MESSAGE = 7;
|
|
655
733
|
// Channel types we map to conversation scopes: DM (1) / group DM (3) = private;
|
|
656
734
|
// everything else (guild text channels, threads) = channel.
|
|
657
735
|
const DISCORD_CHANNEL_TYPE_DM = 1;
|
|
@@ -679,6 +757,8 @@ const WHATSAPP_API_BASE = 'https://graph.facebook.com';
|
|
|
679
757
|
const WHATSAPP_API_VERSION = 'v25.0';
|
|
680
758
|
// A WhatsApp text message body caps at 4096 chars; we chunk a bit under that.
|
|
681
759
|
const WHATSAPP_MESSAGE_LIMIT = 4000;
|
|
760
|
+
// The body of an interactive (reply-button) message holds at most this much.
|
|
761
|
+
const WHATSAPP_INTERACTIVE_BODY_LIMIT = 1024;
|
|
682
762
|
const WHATSAPP_SIGNATURE_HEADER = 'x-hub-signature-256';
|
|
683
763
|
const WHATSAPP_SIGNATURE_PREFIX = 'sha256=';
|
|
684
764
|
// GET-handshake query params + the expected hub.mode value.
|
|
@@ -707,6 +787,24 @@ const EMBEDDING_DIMENSIONS = 1536;
|
|
|
707
787
|
const CHUNK_TARGET_CHARS = 2000;
|
|
708
788
|
const CHUNK_OVERLAP_CHARS = 200;
|
|
709
789
|
const EMBED_BATCH_SIZE = 96;
|
|
790
|
+
// Hybrid resource search. Each ranker contributes its top candidates, and the
|
|
791
|
+
// two lists are merged with reciprocal rank fusion: score = sum of 1/(k + rank).
|
|
792
|
+
// Fusion reads ranks, never raw scores, so a cosine distance and a ts_rank never
|
|
793
|
+
// have to be put on one scale. k = 60 is the constant from the original RRF
|
|
794
|
+
// paper; it keeps one list's #1 from outweighing a chunk both lists agree on.
|
|
795
|
+
const RESOURCE_SEARCH_CANDIDATES = 50;
|
|
796
|
+
const RESOURCE_SEARCH_RRF_K = 60;
|
|
797
|
+
// A query word found in more than 1 of every 10 chunks of the artifact is
|
|
798
|
+
// dropped from the lexical query — never below 5 chunks, so a small artifact
|
|
799
|
+
// keeps its words.
|
|
800
|
+
const RESOURCE_SEARCH_COMMON_TERM_DIVISOR = 10;
|
|
801
|
+
const RESOURCE_SEARCH_COMMON_TERM_MIN = 5;
|
|
802
|
+
// Second stage, on wherever the Worker has an AI binding: the top fused
|
|
803
|
+
// candidates are re-scored by a cross-encoder, which reads query and chunk
|
|
804
|
+
// together instead of comparing two independently computed vectors. One model
|
|
805
|
+
// call per search; a failure falls back to the fused order.
|
|
806
|
+
const RESOURCE_SEARCH_RERANK_CANDIDATES = 30;
|
|
807
|
+
const RESOURCE_SEARCH_RERANK_MODEL = '@cf/baai/bge-reranker-base';
|
|
710
808
|
const RESOURCE_HANDLER_SLEEP_AFTER = '10m';
|
|
711
809
|
const DOCS_URL = 'https://docs.ganju.ai';
|
|
712
810
|
const BASE64_DATA_URI_RE = /data:image\/[a-zA-Z0-9+\-.]+;base64,[A-Za-z0-9+/=\s]+/g;
|
|
@@ -1011,8 +1109,7 @@ const MCP_PROXY_TOOL_NAME_MAX = 64;
|
|
|
1011
1109
|
//
|
|
1012
1110
|
// The row's config holds only `activeVersionId` plus the egress/limit settings;
|
|
1013
1111
|
// names and schemas live on the version. That's what keeps MCP boot off the
|
|
1014
|
-
// dispatcher — a slow or broken script must never break tools/list.
|
|
1015
|
-
// docs/CUSTOM_TOOLS.md ("Boot contract").
|
|
1112
|
+
// dispatcher — a slow or broken script must never break tools/list.
|
|
1016
1113
|
const TOOL_DEFINITION_KEY_CUSTOM_CODE = 'custom-code';
|
|
1017
1114
|
// A version is uploaded as `draft`, becomes `published` when it is made active,
|
|
1018
1115
|
// and is `archived` when a later version replaces it. Exactly one version per
|
|
@@ -1692,7 +1789,8 @@ const PLAN_LIMITS = {
|
|
|
1692
1789
|
includedToolCalls: 0,
|
|
1693
1790
|
toolCallHardCap: 10_000,
|
|
1694
1791
|
includedMessages: 100,
|
|
1695
|
-
includedEmbeddedBytes: 5 * MB
|
|
1792
|
+
includedEmbeddedBytes: 5 * MB,
|
|
1793
|
+
autoSyncIntervals: []
|
|
1696
1794
|
},
|
|
1697
1795
|
PRO: {
|
|
1698
1796
|
maxProjects: null,
|
|
@@ -1715,7 +1813,11 @@ const PLAN_LIMITS = {
|
|
|
1715
1813
|
includedToolCalls: PRICING_INCLUDED_TOOL_CALLS,
|
|
1716
1814
|
toolCallHardCap: PRICING_TOOL_CALL_HARD_CAP,
|
|
1717
1815
|
includedMessages: PRICING_INCLUDED_MESSAGES,
|
|
1718
|
-
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
|
|
1816
|
+
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
|
|
1817
|
+
autoSyncIntervals: [
|
|
1818
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
1819
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
1820
|
+
]
|
|
1719
1821
|
},
|
|
1720
1822
|
ENTERPRISE: {
|
|
1721
1823
|
maxProjects: null,
|
|
@@ -1738,7 +1840,11 @@ const PLAN_LIMITS = {
|
|
|
1738
1840
|
// contract; don't make one plan's abuse limit another plan's product limit.
|
|
1739
1841
|
toolCallHardCap: null,
|
|
1740
1842
|
includedMessages: PRICING_INCLUDED_MESSAGES,
|
|
1741
|
-
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
|
|
1843
|
+
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
|
|
1844
|
+
autoSyncIntervals: [
|
|
1845
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
1846
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
1847
|
+
]
|
|
1742
1848
|
}
|
|
1743
1849
|
};
|
|
1744
1850
|
// Quota features — used as the `feature` discriminator on PlanLimitError so the
|
|
@@ -1756,6 +1862,7 @@ const PLAN_FEATURE_RAW_STORAGE = 'rawStorage';
|
|
|
1756
1862
|
const PLAN_FEATURE_EMBEDDED_STORAGE = 'embeddedStorage';
|
|
1757
1863
|
const PLAN_FEATURE_MESSAGE = 'message';
|
|
1758
1864
|
const PLAN_FEATURE_TOOL_CALL = 'toolCall';
|
|
1865
|
+
const PLAN_FEATURE_RESOURCE_SYNC = 'resourceSync';
|
|
1759
1866
|
// Stable code returned on a quota block so clients can branch on it (402).
|
|
1760
1867
|
const PLAN_LIMIT_ERROR_CODE = 'PLAN_LIMIT_EXCEEDED';
|
|
1761
1868
|
// Billing meter event names. The metering cron reports per-period OVERAGE
|
|
@@ -1923,6 +2030,7 @@ exports.constants = {
|
|
|
1923
2030
|
PLAN_FEATURE_EMBEDDED_STORAGE,
|
|
1924
2031
|
PLAN_FEATURE_MESSAGE,
|
|
1925
2032
|
PLAN_FEATURE_TOOL_CALL,
|
|
2033
|
+
PLAN_FEATURE_RESOURCE_SYNC,
|
|
1926
2034
|
PLAN_LIMIT_ERROR_CODE,
|
|
1927
2035
|
BILLING_METER_MESSAGES,
|
|
1928
2036
|
BILLING_METER_SHARED_MESSAGES,
|
|
@@ -2117,6 +2225,15 @@ exports.constants = {
|
|
|
2117
2225
|
RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER,
|
|
2118
2226
|
RESOURCE_SOURCE_TYPE_CUSTOM_CODE,
|
|
2119
2227
|
RESOURCE_SOURCE_TYPES,
|
|
2228
|
+
RESOURCE_SYNC_INTERVAL_OFF,
|
|
2229
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
2230
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY,
|
|
2231
|
+
RESOURCE_SYNC_INTERVALS,
|
|
2232
|
+
RESOURCE_SYNC_INTERVAL_DEFAULT,
|
|
2233
|
+
RESOURCE_SYNC_INTERVAL_MS,
|
|
2234
|
+
RESOURCE_SYNC_BATCH_SIZE,
|
|
2235
|
+
RESOURCE_SYNC_MANUAL_COOLDOWN_MS,
|
|
2236
|
+
RESOURCE_SYNC_STALE_MS,
|
|
2120
2237
|
CRAWL_RENDERER_CHEERIO,
|
|
2121
2238
|
CRAWL_RENDERER_PLAYWRIGHT,
|
|
2122
2239
|
CRAWL_RENDERERS,
|
|
@@ -2175,6 +2292,11 @@ exports.constants = {
|
|
|
2175
2292
|
SHARED_KEY_HISTORY_LIMIT,
|
|
2176
2293
|
SHARED_KEY_MAX_TOOL_LOOPS,
|
|
2177
2294
|
CHANNEL_MAX_TOOLS,
|
|
2295
|
+
TOOL_CONFIRMATION_TTL_MS,
|
|
2296
|
+
TOOL_CONFIRMATION_REQUIRED_RESULT,
|
|
2297
|
+
TOOL_CONFIRMATION_FALLBACK_QUESTION,
|
|
2298
|
+
TOOL_CONFIRMATION_ASK_INSTRUCTION,
|
|
2299
|
+
TOOL_CONFIRMATION_SYSTEM_NOTE,
|
|
2178
2300
|
CHANNEL_DEBOUNCE_DEFAULT_MS,
|
|
2179
2301
|
CHANNEL_DEBOUNCE_MIN_MS,
|
|
2180
2302
|
CHANNEL_DEBOUNCE_MAX_MS,
|
|
@@ -2207,8 +2329,10 @@ exports.constants = {
|
|
|
2207
2329
|
DISCORD_TIMESTAMP_HEADER,
|
|
2208
2330
|
DISCORD_INTERACTION_TYPE_PING,
|
|
2209
2331
|
DISCORD_INTERACTION_TYPE_APPLICATION_COMMAND,
|
|
2332
|
+
DISCORD_INTERACTION_TYPE_MESSAGE_COMPONENT,
|
|
2210
2333
|
DISCORD_INTERACTION_RESPONSE_PONG,
|
|
2211
2334
|
DISCORD_INTERACTION_RESPONSE_DEFERRED,
|
|
2335
|
+
DISCORD_INTERACTION_RESPONSE_UPDATE_MESSAGE,
|
|
2212
2336
|
DISCORD_CHANNEL_TYPE_DM,
|
|
2213
2337
|
DISCORD_CHANNEL_TYPE_GROUP_DM,
|
|
2214
2338
|
DISCORD_GATEWAY_OP_DISPATCH,
|
|
@@ -2223,6 +2347,7 @@ exports.constants = {
|
|
|
2223
2347
|
WHATSAPP_API_BASE,
|
|
2224
2348
|
WHATSAPP_API_VERSION,
|
|
2225
2349
|
WHATSAPP_MESSAGE_LIMIT,
|
|
2350
|
+
WHATSAPP_INTERACTIVE_BODY_LIMIT,
|
|
2226
2351
|
WHATSAPP_SIGNATURE_HEADER,
|
|
2227
2352
|
WHATSAPP_SIGNATURE_PREFIX,
|
|
2228
2353
|
WHATSAPP_HUB_MODE_PARAM,
|
|
@@ -2238,6 +2363,12 @@ exports.constants = {
|
|
|
2238
2363
|
CHUNK_TARGET_CHARS,
|
|
2239
2364
|
CHUNK_OVERLAP_CHARS,
|
|
2240
2365
|
EMBED_BATCH_SIZE,
|
|
2366
|
+
RESOURCE_SEARCH_CANDIDATES,
|
|
2367
|
+
RESOURCE_SEARCH_RRF_K,
|
|
2368
|
+
RESOURCE_SEARCH_COMMON_TERM_DIVISOR,
|
|
2369
|
+
RESOURCE_SEARCH_COMMON_TERM_MIN,
|
|
2370
|
+
RESOURCE_SEARCH_RERANK_CANDIDATES,
|
|
2371
|
+
RESOURCE_SEARCH_RERANK_MODEL,
|
|
2241
2372
|
RESOURCE_HANDLER_SLEEP_AFTER,
|
|
2242
2373
|
DOCS_URL,
|
|
2243
2374
|
BASE64_DATA_URI_RE,
|