@ganju/utils 0.0.9 → 0.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,17 @@ export interface BufferedChannelMessage {
2
2
  text: string;
3
3
  externalMessageId: string | null;
4
4
  receivedAt: number;
5
+ immediate?: boolean;
6
+ command?: BufferedChannelCommand | null;
7
+ reply?: Record<string, unknown> | null;
5
8
  }
9
+ export interface BufferedChannelCommand {
10
+ promptId: string;
11
+ artifactPromptId: string | null;
12
+ promptTitle: string;
13
+ args: Record<string, string>;
14
+ }
15
+ export declare const batchCommand: (messages: BufferedChannelMessage[]) => BufferedChannelCommand | null;
6
16
  export interface ChannelBufferEnvelope {
7
17
  channelId: string;
8
18
  platform: string;
@@ -17,6 +27,7 @@ export interface ChannelBufferEnvelope {
17
27
  export interface ChannelBufferFlush {
18
28
  envelope: ChannelBufferEnvelope;
19
29
  messages: BufferedChannelMessage[];
30
+ batchId: string;
20
31
  }
21
32
  export declare const resolveDebounceMs: (config: unknown) => number;
22
33
  export declare const channelBufferKey: (channelId: string, externalConversationId: string, externalParticipantId: string) => string;
@@ -1 +1 @@
1
- {"version":3,"file":"channelDebounce.d.ts","sourceRoot":"","sources":["../src/channelDebounce.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;CACpB;AAOD,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACpD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAGD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,qBAAqB,CAAC;IAChC,QAAQ,EAAE,sBAAsB,EAAE,CAAC;CACpC;AAOD,eAAO,MAAM,iBAAiB,GAAI,QAAQ,OAAO,KAAG,MAYnD,CAAC;AAKF,eAAO,MAAM,gBAAgB,GAC3B,WAAW,MAAM,EACjB,wBAAwB,MAAM,EAC9B,uBAAuB,MAAM,KAC5B,MAA2E,CAAC;AAI/E,eAAO,MAAM,oBAAoB,GAC/B,UAAU,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,KAChC,MAIuC,CAAC"}
1
+ {"version":3,"file":"channelDebounce.d.ts","sourceRoot":"","sources":["../src/channelDebounce.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IAKnB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,OAAO,CAAC,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAKxC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACxC;AAID,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAID,eAAO,MAAM,YAAY,GACvB,UAAU,sBAAsB,EAAE,KACjC,sBAAsB,GAAG,IAM3B,CAAC;AAOF,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACpD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAGD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,qBAAqB,CAAC;IAChC,QAAQ,EAAE,sBAAsB,EAAE,CAAC;IAGnC,OAAO,EAAE,MAAM,CAAC;CACjB;AAOD,eAAO,MAAM,iBAAiB,GAAI,QAAQ,OAAO,KAAG,MAYnD,CAAC;AAKF,eAAO,MAAM,gBAAgB,GAC3B,WAAW,MAAM,EACjB,wBAAwB,MAAM,EAC9B,uBAAuB,MAAM,KAC5B,MAA2E,CAAC;AAI/E,eAAO,MAAM,oBAAoB,GAC/B,UAAU,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,KAChC,MAIuC,CAAC"}
@@ -1,7 +1,18 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.joinBufferedMessages = exports.channelBufferKey = exports.resolveDebounceMs = void 0;
3
+ exports.joinBufferedMessages = exports.channelBufferKey = exports.resolveDebounceMs = exports.batchCommand = void 0;
4
4
  const constants_1 = require("./constants");
5
+ // The prompt a batch runs: the newest command in it, since that is the one the
6
+ // participant asked for last. Null for a batch of plain text.
7
+ const batchCommand = (messages) => {
8
+ for (let i = messages.length - 1; i >= 0; i--) {
9
+ const command = messages[i].command;
10
+ if (command)
11
+ return command;
12
+ }
13
+ return null;
14
+ };
15
+ exports.batchCommand = batchCommand;
5
16
  // Read the channel's debounce window off its `config`, clamped to the supported
6
17
  // range. An unset value takes the default; an explicit 0 disables buffering, so
7
18
  // the channel answers every message as it arrives. Anything malformed (a string,
@@ -6,6 +6,17 @@ export declare const chunkText: (text: string, options?: {
6
6
  targetChars?: number;
7
7
  overlapChars?: number;
8
8
  }) => string[];
9
+ export type HeadingSyntax = 'markdown' | 'html';
10
+ /**
11
+ * Which heading syntax a document is written in, or null when it has none we
12
+ * can trust. Plain text is left out on purpose: a line that happens to start
13
+ * with "# " there is as likely a comment or a list as a title.
14
+ */
15
+ export declare const headingSyntaxFor: (input: {
16
+ mimeType?: string | null;
17
+ fileName?: string | null;
18
+ isWebPage?: boolean;
19
+ }) => HeadingSyntax | null;
9
20
  export declare const buildHeader: (input: {
10
21
  title?: string | null;
11
22
  description?: string | null;
@@ -13,9 +24,10 @@ export declare const buildHeader: (input: {
13
24
  mimeType?: string | null;
14
25
  fileName?: string | null;
15
26
  }) => string;
16
- export interface ChunkMetadata extends ExtractedDocumentMetadata {
17
- chunkIndexInPage: number;
18
- chunksInPage: number;
27
+ export interface ChunkMetadata extends Partial<ExtractedDocumentMetadata> {
28
+ chunkIndexInPage?: number;
29
+ chunksInPage?: number;
30
+ headingPath?: string[];
19
31
  }
20
32
  export interface PreparedChunk {
21
33
  content: string;
@@ -24,5 +36,7 @@ export interface PreparedChunk {
24
36
  export declare const prepareChunks: (header: string, documents: ExtractedDocument[] | null, fallbackContent: string | null, options?: {
25
37
  targetChars?: number;
26
38
  overlapChars?: number;
39
+ mimeType?: string | null;
40
+ fileName?: string | null;
27
41
  }) => PreparedChunk[];
28
42
  //# sourceMappingURL=chunking.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,KAAK,EACV,iBAAiB,EACjB,yBAAyB,EAC1B,MAAM,qBAAqB,CAAC;AAE7B,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,SAAS,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AASpE,eAAO,MAAM,cAAc,GACzB,MAAM,MAAM,EACZ,QAAQ,MAAM,EACd,aAAY,SAAS,SAAS,EAA+B,KAC5D,MAAM,EA8BR,CAAC;AAEF,eAAO,MAAM,SAAS,GACpB,MAAM,MAAM,EACZ,UAAU;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KACxD,MAAM,EAeR,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO;IACjC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KAAG,MASW,CAAC;AAEhB,MAAM,WAAW,aAAc,SAAQ,yBAAyB;IAC9D,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CAChC;AAED,eAAO,MAAM,aAAa,GACxB,QAAQ,MAAM,EACd,WAAW,iBAAiB,EAAE,GAAG,IAAI,EACrC,iBAAiB,MAAM,GAAG,IAAI,EAC9B,UAAU;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KACxD,aAAa,EAgCf,CAAC"}
1
+ {"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,KAAK,EACV,iBAAiB,EACjB,yBAAyB,EAC1B,MAAM,qBAAqB,CAAC;AAE7B,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,SAAS,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AASpE,eAAO,MAAM,cAAc,GACzB,MAAM,MAAM,EACZ,QAAQ,MAAM,EACd,aAAY,SAAS,SAAS,EAA+B,KAC5D,MAAM,EA8BR,CAAC;AA4BF,eAAO,MAAM,SAAS,GACpB,MAAM,MAAM,EACZ,UAAU;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KACxD,MAAM,EAAoD,CAAC;AAE9D,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,MAAM,CAAC;AAehD;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,GAAI,OAAO;IACtC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,KAAG,aAAa,GAAG,IAUnB,CAAC;AAuEF,eAAO,MAAM,WAAW,GAAI,OAAO;IACjC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KAAG,MASW,CAAC;AAIhB,MAAM,WAAW,aAAc,SAAQ,OAAO,CAAC,yBAAyB,CAAC;IACvE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CAChC;AAID,eAAO,MAAM,aAAa,GACxB,QAAQ,MAAM,EACd,WAAW,iBAAiB,EAAE,GAAG,IAAI,EACrC,iBAAiB,MAAM,GAAG,IAAI,EAC9B,UAAU;IACR,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KACA,aAAa,EA4Cf,CAAC"}
package/dist/chunking.js CHANGED
@@ -1,7 +1,8 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.prepareChunks = exports.buildHeader = exports.chunkText = exports.splitRecursive = void 0;
3
+ exports.prepareChunks = exports.buildHeader = exports.headingSyntaxFor = exports.chunkText = exports.splitRecursive = void 0;
4
4
  const constants_1 = require("./constants");
5
+ const sanitize_1 = require("./sanitize");
5
6
  const trySplit = (text, sep) => {
6
7
  if (typeof sep === 'string') {
7
8
  return text.includes(sep) ? text.split(sep) : null;
@@ -40,23 +41,115 @@ const splitRecursive = (text, target, separators = constants_1.constants.CHUNK_S
40
41
  return chunks;
41
42
  };
42
43
  exports.splitRecursive = splitRecursive;
43
- const chunkText = (text, options) => {
44
+ // Each piece keeps its `raw` text — the slice of the input it covers, without
45
+ // the overlap borrowed from the piece before — so a caller can walk the input
46
+ // in order without reading any of it twice.
47
+ const chunkPieces = (text, options) => {
44
48
  const target = options?.targetChars ?? constants_1.constants.CHUNK_TARGET_CHARS;
45
49
  const overlap = options?.overlapChars ?? constants_1.constants.CHUNK_OVERLAP_CHARS;
46
50
  const trimmed = text.trim();
47
51
  if (!trimmed)
48
52
  return [];
49
53
  if (trimmed.length <= target)
50
- return [trimmed];
54
+ return [{ raw: trimmed, content: trimmed }];
51
55
  const rawChunks = (0, exports.splitRecursive)(trimmed, target);
52
- const withOverlap = [];
56
+ const pieces = [];
53
57
  for (let i = 0; i < rawChunks.length; i++) {
54
58
  const prev = i > 0 ? rawChunks[i - 1].slice(-overlap) : '';
55
- withOverlap.push((prev ? prev + '\n' : '') + rawChunks[i]);
59
+ pieces.push({
60
+ raw: rawChunks[i],
61
+ content: (prev ? prev + '\n' : '') + rawChunks[i]
62
+ });
56
63
  }
57
- return withOverlap.filter(c => c.trim().length > 0);
64
+ return pieces.filter(p => p.content.trim().length > 0);
58
65
  };
66
+ const chunkText = (text, options) => chunkPieces(text, options).map(p => p.content);
59
67
  exports.chunkText = chunkText;
68
+ const MARKDOWN_HEADING = /^ {0,3}(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/;
69
+ const MARKDOWN_FENCE = /^ {0,3}(```|~~~)/;
70
+ const HTML_HEADING = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi;
71
+ const MAX_HEADING_CHARS = 120;
72
+ const MARKDOWN_FILE_NAME = /\.(md|markdown|mdx)$/i;
73
+ /**
74
+ * Which heading syntax a document is written in, or null when it has none we
75
+ * can trust. Plain text is left out on purpose: a line that happens to start
76
+ * with "# " there is as likely a comment or a list as a title.
77
+ */
78
+ const headingSyntaxFor = (input) => {
79
+ // Crawled pages are flattened to text with their headings written back as
80
+ // Markdown lines, whatever the page's own content type was.
81
+ if (input.isWebPage)
82
+ return 'markdown';
83
+ if (input.mimeType === constants_1.constants.MIMETYPE_TEXT_MARKDOWN)
84
+ return 'markdown';
85
+ if (input.mimeType === constants_1.constants.MIMETYPE_TEXT_HTML)
86
+ return 'html';
87
+ if (input.fileName && MARKDOWN_FILE_NAME.test(input.fileName)) {
88
+ return 'markdown';
89
+ }
90
+ return null;
91
+ };
92
+ exports.headingSyntaxFor = headingSyntaxFor;
93
+ const cleanHeading = (text) => text.replace(/\s+/g, ' ').trim().slice(0, MAX_HEADING_CHARS);
94
+ const scanMarkdownHeadings = (text, state) => {
95
+ const events = [];
96
+ let offset = 0;
97
+ for (const line of text.split('\n')) {
98
+ // A `#` inside a fenced block is a shell comment or a preprocessor line,
99
+ // not a title — and a fence can open in one chunk and close in the next.
100
+ if (MARKDOWN_FENCE.test(line)) {
101
+ state.inFence = !state.inFence;
102
+ }
103
+ else if (!state.inFence) {
104
+ const match = MARKDOWN_HEADING.exec(line);
105
+ const heading = match ? cleanHeading(match[2]) : '';
106
+ if (match && heading) {
107
+ events.push({ offset, level: match[1].length, text: heading });
108
+ }
109
+ }
110
+ offset += line.length + 1;
111
+ }
112
+ return events;
113
+ };
114
+ const scanHtmlHeadings = (text) => [...text.matchAll(HTML_HEADING)]
115
+ .map(match => ({
116
+ offset: match.index ?? 0,
117
+ level: Number(match[1]),
118
+ text: cleanHeading((0, sanitize_1.decodeEntities)(match[2].replace(/<[^>]*>/g, ' ')))
119
+ }))
120
+ .filter(event => event.text);
121
+ /**
122
+ * Follows the heading hierarchy across a document's chunks, fed in order.
123
+ * Returns, for each chunk, the path of headings it sits under — the section a
124
+ * reader would be pointed to. A chunk that opens with a heading belongs to that
125
+ * heading; one that starts mid-section belongs to the section it continues,
126
+ * even when a later heading begins inside it.
127
+ */
128
+ const createHeadingTracker = (syntax) => {
129
+ let stack = [];
130
+ const fence = { inFence: false };
131
+ const apply = (event) => {
132
+ stack = stack.filter(h => h.level < event.level);
133
+ stack.push({ level: event.level, text: event.text });
134
+ };
135
+ return (raw) => {
136
+ const events = syntax === 'markdown'
137
+ ? scanMarkdownHeadings(raw, fence)
138
+ : scanHtmlHeadings(raw);
139
+ const [first, ...rest] = events;
140
+ // Before a document's first heading there is no section to continue — only
141
+ // the metadata header, or the markup around an HTML body — so a chunk that
142
+ // holds the first heading belongs to it wherever in the chunk it falls.
143
+ const opensWithHeading = first !== undefined &&
144
+ (stack.length === 0 || raw.slice(0, first.offset).trim() === '');
145
+ if (opensWithHeading)
146
+ apply(first);
147
+ const path = stack.map(h => h.text);
148
+ for (const event of opensWithHeading ? rest : events)
149
+ apply(event);
150
+ return path;
151
+ };
152
+ };
60
153
  const buildHeader = (input) => [
61
154
  input.title ? `Title: ${input.title}` : null,
62
155
  input.description ? `Description: ${input.description}` : null,
@@ -67,37 +160,50 @@ const buildHeader = (input) => [
67
160
  .filter(Boolean)
68
161
  .join('\n');
69
162
  exports.buildHeader = buildHeader;
163
+ const CONTROL_CHARS = /[\x00-\x08\x0B\x0C\x0E-\x1F]/g;
70
164
  const prepareChunks = (header, documents, fallbackContent, options) => {
71
165
  if (documents && documents.length > 0) {
72
166
  const out = [];
73
167
  documents.forEach((doc, docIndex) => {
74
- const body = (doc.pageContent || '')
75
- .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
76
- .trim();
168
+ const body = (doc.pageContent || '').replace(CONTROL_CHARS, '').trim();
77
169
  if (!body)
78
170
  return;
79
171
  const text = docIndex === 0 && header ? `${header}\n\n${body}` : body;
80
- const pieces = (0, exports.chunkText)(text, options);
172
+ const syntax = (0, exports.headingSyntaxFor)({
173
+ mimeType: doc.metadata?.source?.mimeType ?? options?.mimeType,
174
+ fileName: doc.metadata?.source?.fileName ?? options?.fileName,
175
+ isWebPage: !!doc.metadata?.web
176
+ });
177
+ const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
178
+ const pieces = chunkPieces(text, options);
81
179
  pieces.forEach((piece, pieceIndex) => {
180
+ const headingPath = headingPathOf?.(piece.raw) ?? [];
82
181
  out.push({
83
- content: piece,
182
+ content: piece.content,
84
183
  metadata: {
85
184
  ...doc.metadata,
86
185
  chunkIndexInPage: pieceIndex,
87
- chunksInPage: pieces.length
186
+ chunksInPage: pieces.length,
187
+ ...(headingPath.length > 0 ? { headingPath } : {})
88
188
  }
89
189
  });
90
190
  });
91
191
  });
92
192
  return out;
93
193
  }
94
- const body = (fallbackContent || '')
95
- .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
96
- .trim();
194
+ const body = (fallbackContent || '').replace(CONTROL_CHARS, '').trim();
97
195
  const fullText = [header, body].filter(Boolean).join('\n\n');
98
- return (0, exports.chunkText)(fullText, options).map(content => ({
99
- content,
100
- metadata: null
101
- }));
196
+ const syntax = (0, exports.headingSyntaxFor)({
197
+ mimeType: options?.mimeType,
198
+ fileName: options?.fileName
199
+ });
200
+ const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
201
+ return chunkPieces(fullText, options).map(piece => {
202
+ const headingPath = headingPathOf?.(piece.raw) ?? [];
203
+ return {
204
+ content: piece.content,
205
+ metadata: headingPath.length > 0 ? { headingPath } : null
206
+ };
207
+ });
102
208
  };
103
209
  exports.prepareChunks = prepareChunks;
@@ -6,6 +6,8 @@ declare const ONE_DRIVE_TAB_MY_FILES: "my-files";
6
6
  declare const ONE_DRIVE_TAB_SHARED_WITH_ME: "shared-with-me";
7
7
  declare const ONE_DRIVE_TAB_RECENT: "recent";
8
8
  declare const ONE_DRIVE_TAB_DRIVES: "drives";
9
+ declare const RESOURCE_SYNC_INTERVAL_DAILY: "daily";
10
+ declare const RESOURCE_SYNC_INTERVAL_WEEKLY: "weekly";
9
11
  declare const LLM_PROVIDER_ANTHROPIC: "anthropic";
10
12
  declare const LLM_PROVIDER_OPENAI: "openai";
11
13
  declare const LLM_PROVIDER_GOOGLE: "google";
@@ -59,6 +61,7 @@ interface PlanLimits {
59
61
  toolCallHardCap: number | null;
60
62
  includedMessages: number;
61
63
  includedEmbeddedBytes: number;
64
+ autoSyncIntervals: ReadonlyArray<typeof RESOURCE_SYNC_INTERVAL_DAILY | typeof RESOURCE_SYNC_INTERVAL_WEEKLY>;
62
65
  }
63
66
  export type { PlanLimits };
64
67
  export declare const constants: {
@@ -104,6 +107,7 @@ export declare const constants: {
104
107
  PLAN_FEATURE_EMBEDDED_STORAGE: "embeddedStorage";
105
108
  PLAN_FEATURE_MESSAGE: "message";
106
109
  PLAN_FEATURE_TOOL_CALL: "toolCall";
110
+ PLAN_FEATURE_RESOURCE_SYNC: "resourceSync";
107
111
  PLAN_LIMIT_ERROR_CODE: string;
108
112
  BILLING_METER_MESSAGES: string;
109
113
  BILLING_METER_SHARED_MESSAGES: string;
@@ -309,6 +313,18 @@ export declare const constants: {
309
313
  RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER: "ONE_DRIVE_FOLDER";
310
314
  RESOURCE_SOURCE_TYPE_CUSTOM_CODE: "CUSTOM_CODE";
311
315
  RESOURCE_SOURCE_TYPES: ("FILE" | "WEBSITE" | "GOOGLE_DRIVE_FOLDER" | "ONE_DRIVE_FOLDER" | "CUSTOM_CODE")[];
316
+ RESOURCE_SYNC_INTERVAL_OFF: "off";
317
+ RESOURCE_SYNC_INTERVAL_DAILY: "daily";
318
+ RESOURCE_SYNC_INTERVAL_WEEKLY: "weekly";
319
+ RESOURCE_SYNC_INTERVALS: readonly ["off", "daily", "weekly"];
320
+ RESOURCE_SYNC_INTERVAL_DEFAULT: "weekly";
321
+ RESOURCE_SYNC_INTERVAL_MS: {
322
+ readonly daily: number;
323
+ readonly weekly: number;
324
+ };
325
+ RESOURCE_SYNC_BATCH_SIZE: number;
326
+ RESOURCE_SYNC_MANUAL_COOLDOWN_MS: number;
327
+ RESOURCE_SYNC_STALE_MS: number;
312
328
  CRAWL_RENDERER_CHEERIO: "cheerio";
313
329
  CRAWL_RENDERER_PLAYWRIGHT: "playwright";
314
330
  CRAWL_RENDERERS: ("cheerio" | "playwright")[];
@@ -371,6 +387,11 @@ export declare const constants: {
371
387
  SHARED_KEY_HISTORY_LIMIT: number;
372
388
  SHARED_KEY_MAX_TOOL_LOOPS: number;
373
389
  CHANNEL_MAX_TOOLS: number;
390
+ TOOL_CONFIRMATION_TTL_MS: number;
391
+ TOOL_CONFIRMATION_REQUIRED_RESULT: string;
392
+ TOOL_CONFIRMATION_FALLBACK_QUESTION: string;
393
+ TOOL_CONFIRMATION_ASK_INSTRUCTION: string;
394
+ TOOL_CONFIRMATION_SYSTEM_NOTE: string;
374
395
  CHANNEL_DEBOUNCE_DEFAULT_MS: number;
375
396
  CHANNEL_DEBOUNCE_MIN_MS: number;
376
397
  CHANNEL_DEBOUNCE_MAX_MS: number;
@@ -403,8 +424,10 @@ export declare const constants: {
403
424
  DISCORD_TIMESTAMP_HEADER: string;
404
425
  DISCORD_INTERACTION_TYPE_PING: number;
405
426
  DISCORD_INTERACTION_TYPE_APPLICATION_COMMAND: number;
427
+ DISCORD_INTERACTION_TYPE_MESSAGE_COMPONENT: number;
406
428
  DISCORD_INTERACTION_RESPONSE_PONG: number;
407
429
  DISCORD_INTERACTION_RESPONSE_DEFERRED: number;
430
+ DISCORD_INTERACTION_RESPONSE_UPDATE_MESSAGE: number;
408
431
  DISCORD_CHANNEL_TYPE_DM: number;
409
432
  DISCORD_CHANNEL_TYPE_GROUP_DM: number;
410
433
  DISCORD_GATEWAY_OP_DISPATCH: number;
@@ -419,6 +442,7 @@ export declare const constants: {
419
442
  WHATSAPP_API_BASE: string;
420
443
  WHATSAPP_API_VERSION: string;
421
444
  WHATSAPP_MESSAGE_LIMIT: number;
445
+ WHATSAPP_INTERACTIVE_BODY_LIMIT: number;
422
446
  WHATSAPP_SIGNATURE_HEADER: string;
423
447
  WHATSAPP_SIGNATURE_PREFIX: string;
424
448
  WHATSAPP_HUB_MODE_PARAM: string;
@@ -434,6 +458,12 @@ export declare const constants: {
434
458
  CHUNK_TARGET_CHARS: number;
435
459
  CHUNK_OVERLAP_CHARS: number;
436
460
  EMBED_BATCH_SIZE: number;
461
+ RESOURCE_SEARCH_CANDIDATES: number;
462
+ RESOURCE_SEARCH_RRF_K: number;
463
+ RESOURCE_SEARCH_COMMON_TERM_DIVISOR: number;
464
+ RESOURCE_SEARCH_COMMON_TERM_MIN: number;
465
+ RESOURCE_SEARCH_RERANK_CANDIDATES: number;
466
+ RESOURCE_SEARCH_RERANK_MODEL: string;
437
467
  RESOURCE_HANDLER_SLEEP_AFTER: string;
438
468
  DOCS_URL: string;
439
469
  BASE64_DATA_URI_RE: RegExp;
@@ -1 +1 @@
1
- {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AA6JA,QAAA,MAAM,yBAAyB,EAAiB,UAAU,CAAC;AAC3D,QAAA,MAAM,+BAA+B,EAAuB,gBAAgB,CAAC;AAC7E,QAAA,MAAM,8BAA8B,EAAsB,eAAe,CAAC;AAC1E,QAAA,MAAM,wBAAwB,EAAgB,SAAS,CAAC;AAuCxD,QAAA,MAAM,sBAAsB,EAAiB,UAAU,CAAC;AACxD,QAAA,MAAM,4BAA4B,EAAuB,gBAAgB,CAAC;AAC1E,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAClD,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAmYlD,QAAA,MAAM,sBAAsB,EAAkB,WAAW,CAAC;AAC1D,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAGjD,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAqUjD,MAAM,MAAM,mBAAmB,GAC3B;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC9D;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA2+BpE,UAAU,UAAU;IAElB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAGhC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAMjC,sBAAsB,EAAE,MAAM,CAAC;IAM/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,SAAS,EAAE,OAAO,CAAC;IAKnB,eAAe,EAAE,OAAO,CAAC;IAOzB,gBAAgB,EAAE,OAAO,CAAC;IAK1B,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAQ3C,iBAAiB,EAAE,MAAM,CAAC;IAM1B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAG/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAiPD,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAphEhB,OAAO,yBAAyB,GAChC,OAAO,+BAA+B,GACtC,OAAO,8BAA8B,GACrC,OAAO,wBAAwB;eAC5B,MAAM;;;;;;;;;;;;;;;;;;eAsCT,OAAO,sBAAsB,GAC7B,OAAO,4BAA4B,GACnC,OAAO,oBAAoB,GAC3B,OAAO,oBAAoB;eACxB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA6YT,OAAO,mBAAmB,GAC1B,OAAO,mBAAmB,GAC1B,OAAO,sBAAsB;eAC1B,MAAM;eACN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAgPwB,MAAM,GAAG,MAAM;cAAQ,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA46DnE,CAAC"}
1
+ {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AA6JA,QAAA,MAAM,yBAAyB,EAAiB,UAAU,CAAC;AAC3D,QAAA,MAAM,+BAA+B,EAAuB,gBAAgB,CAAC;AAC7E,QAAA,MAAM,8BAA8B,EAAsB,eAAe,CAAC;AAC1E,QAAA,MAAM,wBAAwB,EAAgB,SAAS,CAAC;AAuCxD,QAAA,MAAM,sBAAsB,EAAiB,UAAU,CAAC;AACxD,QAAA,MAAM,4BAA4B,EAAuB,gBAAgB,CAAC;AAC1E,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAClD,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAoFlD,QAAA,MAAM,4BAA4B,EAAc,OAAO,CAAC;AACxD,QAAA,MAAM,6BAA6B,EAAe,QAAQ,CAAC;AAmV3D,QAAA,MAAM,sBAAsB,EAAkB,WAAW,CAAC;AAC1D,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAGjD,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAyYjD,MAAM,MAAM,mBAAmB,GAC3B;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC9D;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA0+BpE,UAAU,UAAU;IAElB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAGhC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAMjC,sBAAsB,EAAE,MAAM,CAAC;IAM/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,SAAS,EAAE,OAAO,CAAC;IAKnB,eAAe,EAAE,OAAO,CAAC;IAOzB,gBAAgB,EAAE,OAAO,CAAC;IAK1B,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAQ3C,iBAAiB,EAAE,MAAM,CAAC;IAM1B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAG/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAI9B,iBAAiB,EAAE,aAAa,CAC9B,OAAO,4BAA4B,GAAG,OAAO,6BAA6B,CAC3E,CAAC;CACH;AA2PD,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eA5oEhB,OAAO,yBAAyB,GAChC,OAAO,+BAA+B,GACtC,OAAO,8BAA8B,GACrC,OAAO,wBAAwB;eAC5B,MAAM;;;;;;;;;;;;;;;;;;eAsCT,OAAO,sBAAsB,GAC7B,OAAO,4BAA4B,GACnC,OAAO,oBAAoB,GAC3B,OAAO,oBAAoB;eACxB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkbT,OAAO,mBAAmB,GAC1B,OAAO,mBAAmB,GAC1B,OAAO,sBAAsB;eAC1B,MAAM;eACN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAoTwB,MAAM,GAAG,MAAM;cAAQ,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAm9DnE,CAAC"}
package/dist/constants.js CHANGED
@@ -208,6 +208,42 @@ const RESOURCE_SOURCE_TYPES = [
208
208
  RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER,
209
209
  RESOURCE_SOURCE_TYPE_CUSTOM_CODE
210
210
  ];
211
+ // How often a source that lives somewhere else — a crawled website, a Drive or
212
+ // OneDrive folder, a file imported from either — is refreshed from it. Every
213
+ // source is stored with an interval, weekly unless its owner changes it; the
214
+ // hourly sync job only acts on it for a plan that includes automatic sync, so a
215
+ // Free org's sources start syncing the moment it upgrades.
216
+ const RESOURCE_SYNC_INTERVAL_OFF = 'off';
217
+ const RESOURCE_SYNC_INTERVAL_DAILY = 'daily';
218
+ const RESOURCE_SYNC_INTERVAL_WEEKLY = 'weekly';
219
+ const RESOURCE_SYNC_INTERVALS = [
220
+ RESOURCE_SYNC_INTERVAL_OFF,
221
+ RESOURCE_SYNC_INTERVAL_DAILY,
222
+ RESOURCE_SYNC_INTERVAL_WEEKLY
223
+ ];
224
+ const RESOURCE_SYNC_INTERVAL_DEFAULT = RESOURCE_SYNC_INTERVAL_WEEKLY;
225
+ const RESOURCE_SYNC_INTERVAL_MS = {
226
+ [RESOURCE_SYNC_INTERVAL_DAILY]: 24 * 60 * 60 * 1000,
227
+ [RESOURCE_SYNC_INTERVAL_WEEKLY]: 7 * 24 * 60 * 60 * 1000
228
+ };
229
+ // Sources started per hourly run. A sync is cheap to start and expensive to
230
+ // run — a website re-crawls every page — so the job works through a backlog
231
+ // over several hours rather than launching all of it at once.
232
+ const RESOURCE_SYNC_BATCH_SIZE = 50;
233
+ // How soon a source can be synced by hand again. Manual sync is open to every
234
+ // plan, and a website sync re-crawls every page, so it can't be a button to
235
+ // press in a loop.
236
+ const RESOURCE_SYNC_MANUAL_COOLDOWN_MS = 5 * 60 * 1000;
237
+ // How long a sync may stay PENDING before it counts as stuck rather than
238
+ // running. A lost queue message or a job that exhausted its retries never
239
+ // completes its root, and a root that stays PENDING would otherwise be skipped
240
+ // by the hourly job and refused by "Sync now" forever. Twelve hours outlasts
241
+ // any realistic wait: the crawl-page queue runs 5 at a time for every org, and
242
+ // one hourly run can queue 50 sites — thousands of pages, hours of backlog. A
243
+ // shorter window restarts syncs that are merely queued, re-fetching every page
244
+ // into the same backlog; a longer one leaves a stuck daily source missing more
245
+ // than one sync.
246
+ const RESOURCE_SYNC_STALE_MS = 12 * 60 * 60 * 1000;
211
247
  const CRAWL_RENDERER_CHEERIO = 'cheerio';
212
248
  const CRAWL_RENDERER_PLAYWRIGHT = 'playwright';
213
249
  const CRAWL_RENDERERS = [CRAWL_RENDERER_CHEERIO, CRAWL_RENDERER_PLAYWRIGHT];
@@ -594,6 +630,44 @@ const CHANNEL_DEBOUNCE_JOIN = '\n';
594
630
  // and how many times it tries before dropping the batch rather than looping.
595
631
  const CHANNEL_DEBOUNCE_RETRY_MS = 5000;
596
632
  const CHANNEL_DEBOUNCE_MAX_ATTEMPTS = 3;
633
+ // Confirming sensitive actions, for organizations that turn it on. A channel
634
+ // bot that wants to run a tool annotated `destructiveHint` asks first, and the
635
+ // call runs only if the participant's next message is a plain yes.
636
+ //
637
+ // How long the question stays open. An answer after that is read as a new
638
+ // message, never as consent to something asked long ago.
639
+ const TOOL_CONFIRMATION_TTL_MS = 30 * 60 * 1000;
640
+ // The tool result the model gets instead of running the call. It is what makes
641
+ // the model write the question — in the participant's language, naming what
642
+ // the call would do — rather than claiming the action happened.
643
+ // The call's exact arguments are listed under the model's question by the
644
+ // runner, so the model only has to say what it is about to do.
645
+ const TOOL_CONFIRMATION_REQUIRED_RESULT = 'Not run yet: this action needs the user to confirm it first. In one or ' +
646
+ 'two sentences, tell them what it will do (who it goes to, what will be ' +
647
+ 'changed or deleted) and ask them to answer yes to go ahead or no to ' +
648
+ 'cancel — in their own language, never quoting English words at someone ' +
649
+ 'writing in another one. The exact details are listed for them under your ' +
650
+ 'message. It runs only if their next message is yes. Do not say it was done.';
651
+ // Told to the model on every turn of an organization that confirms: the
652
+ // asking is done for it, so it calls the tool when asked instead of asking
653
+ // first itself — and calls every action a request needs in one step, since
654
+ // the turn stops at the first held call and each later one would be a
655
+ // question of its own.
656
+ const TOOL_CONFIRMATION_SYSTEM_NOTE = 'Actions that send, change or delete something are confirmed with the ' +
657
+ 'user by the system before they run. Ask for any detail you are missing, ' +
658
+ 'but once you have what the action needs, call the tool — do not ask the ' +
659
+ 'user to confirm it yourself; the system asks them and runs it on their ' +
660
+ 'yes. When a request needs several such actions (say, delete an event ' +
661
+ 'and create its replacement), call them all in the same step, so the ' +
662
+ 'user confirms them together once.';
663
+ // The system-prompt addition for the step that writes the question, with the
664
+ // held actions' names in place of {{actions}}.
665
+ const TOOL_CONFIRMATION_ASK_INSTRUCTION = 'IMPORTANT: these actions have NOT been done: {{actions}}. They are ' +
666
+ 'waiting for the user to confirm them. Your reply must ask the user to ' +
667
+ 'confirm, and must not say or suggest that they were done, nor describe ' +
668
+ 'or make up any result from them.';
669
+ // The question when the model wrote none, above the runner's own summary.
670
+ const TOOL_CONFIRMATION_FALLBACK_QUESTION = 'Before I do this, please confirm. Reply yes to go ahead, or no to cancel.';
597
671
  const TELEGRAM_SECRET_HEADER = 'x-telegram-bot-api-secret-token';
598
672
  const TELEGRAM_API_BASE = 'https://api.telegram.org';
599
673
  const TELEGRAM_MESSAGE_LIMIT = 3500;
@@ -650,8 +724,12 @@ const DISCORD_TIMESTAMP_HEADER = 'x-signature-timestamp';
650
724
  // Interaction request types and response types (Discord API).
651
725
  const DISCORD_INTERACTION_TYPE_PING = 1;
652
726
  const DISCORD_INTERACTION_TYPE_APPLICATION_COMMAND = 2;
727
+ // A tap on a message button — the Yes/No under a confirmation question.
728
+ const DISCORD_INTERACTION_TYPE_MESSAGE_COMPONENT = 3;
653
729
  const DISCORD_INTERACTION_RESPONSE_PONG = 1;
654
730
  const DISCORD_INTERACTION_RESPONSE_DEFERRED = 5;
731
+ // Edits the message the tapped button is on, in the same response.
732
+ const DISCORD_INTERACTION_RESPONSE_UPDATE_MESSAGE = 7;
655
733
  // Channel types we map to conversation scopes: DM (1) / group DM (3) = private;
656
734
  // everything else (guild text channels, threads) = channel.
657
735
  const DISCORD_CHANNEL_TYPE_DM = 1;
@@ -679,6 +757,8 @@ const WHATSAPP_API_BASE = 'https://graph.facebook.com';
679
757
  const WHATSAPP_API_VERSION = 'v25.0';
680
758
  // A WhatsApp text message body caps at 4096 chars; we chunk a bit under that.
681
759
  const WHATSAPP_MESSAGE_LIMIT = 4000;
760
+ // The body of an interactive (reply-button) message holds at most this much.
761
+ const WHATSAPP_INTERACTIVE_BODY_LIMIT = 1024;
682
762
  const WHATSAPP_SIGNATURE_HEADER = 'x-hub-signature-256';
683
763
  const WHATSAPP_SIGNATURE_PREFIX = 'sha256=';
684
764
  // GET-handshake query params + the expected hub.mode value.
@@ -707,6 +787,24 @@ const EMBEDDING_DIMENSIONS = 1536;
707
787
  const CHUNK_TARGET_CHARS = 2000;
708
788
  const CHUNK_OVERLAP_CHARS = 200;
709
789
  const EMBED_BATCH_SIZE = 96;
790
+ // Hybrid resource search. Each ranker contributes its top candidates, and the
791
+ // two lists are merged with reciprocal rank fusion: score = sum of 1/(k + rank).
792
+ // Fusion reads ranks, never raw scores, so a cosine distance and a ts_rank never
793
+ // have to be put on one scale. k = 60 is the constant from the original RRF
794
+ // paper; it keeps one list's #1 from outweighing a chunk both lists agree on.
795
+ const RESOURCE_SEARCH_CANDIDATES = 50;
796
+ const RESOURCE_SEARCH_RRF_K = 60;
797
+ // A query word found in more than 1 of every 10 chunks of the artifact is
798
+ // dropped from the lexical query — never below 5 chunks, so a small artifact
799
+ // keeps its words.
800
+ const RESOURCE_SEARCH_COMMON_TERM_DIVISOR = 10;
801
+ const RESOURCE_SEARCH_COMMON_TERM_MIN = 5;
802
+ // Second stage, on wherever the Worker has an AI binding: the top fused
803
+ // candidates are re-scored by a cross-encoder, which reads query and chunk
804
+ // together instead of comparing two independently computed vectors. One model
805
+ // call per search; a failure falls back to the fused order.
806
+ const RESOURCE_SEARCH_RERANK_CANDIDATES = 30;
807
+ const RESOURCE_SEARCH_RERANK_MODEL = '@cf/baai/bge-reranker-base';
710
808
  const RESOURCE_HANDLER_SLEEP_AFTER = '10m';
711
809
  const DOCS_URL = 'https://docs.ganju.ai';
712
810
  const BASE64_DATA_URI_RE = /data:image\/[a-zA-Z0-9+\-.]+;base64,[A-Za-z0-9+/=\s]+/g;
@@ -1011,8 +1109,7 @@ const MCP_PROXY_TOOL_NAME_MAX = 64;
1011
1109
  //
1012
1110
  // The row's config holds only `activeVersionId` plus the egress/limit settings;
1013
1111
  // names and schemas live on the version. That's what keeps MCP boot off the
1014
- // dispatcher — a slow or broken script must never break tools/list. See
1015
- // docs/CUSTOM_TOOLS.md ("Boot contract").
1112
+ // dispatcher — a slow or broken script must never break tools/list.
1016
1113
  const TOOL_DEFINITION_KEY_CUSTOM_CODE = 'custom-code';
1017
1114
  // A version is uploaded as `draft`, becomes `published` when it is made active,
1018
1115
  // and is `archived` when a later version replaces it. Exactly one version per
@@ -1692,7 +1789,8 @@ const PLAN_LIMITS = {
1692
1789
  includedToolCalls: 0,
1693
1790
  toolCallHardCap: 10_000,
1694
1791
  includedMessages: 100,
1695
- includedEmbeddedBytes: 5 * MB
1792
+ includedEmbeddedBytes: 5 * MB,
1793
+ autoSyncIntervals: []
1696
1794
  },
1697
1795
  PRO: {
1698
1796
  maxProjects: null,
@@ -1715,7 +1813,11 @@ const PLAN_LIMITS = {
1715
1813
  includedToolCalls: PRICING_INCLUDED_TOOL_CALLS,
1716
1814
  toolCallHardCap: PRICING_TOOL_CALL_HARD_CAP,
1717
1815
  includedMessages: PRICING_INCLUDED_MESSAGES,
1718
- includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
1816
+ includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
1817
+ autoSyncIntervals: [
1818
+ RESOURCE_SYNC_INTERVAL_DAILY,
1819
+ RESOURCE_SYNC_INTERVAL_WEEKLY
1820
+ ]
1719
1821
  },
1720
1822
  ENTERPRISE: {
1721
1823
  maxProjects: null,
@@ -1738,7 +1840,11 @@ const PLAN_LIMITS = {
1738
1840
  // contract; don't make one plan's abuse limit another plan's product limit.
1739
1841
  toolCallHardCap: null,
1740
1842
  includedMessages: PRICING_INCLUDED_MESSAGES,
1741
- includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
1843
+ includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
1844
+ autoSyncIntervals: [
1845
+ RESOURCE_SYNC_INTERVAL_DAILY,
1846
+ RESOURCE_SYNC_INTERVAL_WEEKLY
1847
+ ]
1742
1848
  }
1743
1849
  };
1744
1850
  // Quota features — used as the `feature` discriminator on PlanLimitError so the
@@ -1756,6 +1862,7 @@ const PLAN_FEATURE_RAW_STORAGE = 'rawStorage';
1756
1862
  const PLAN_FEATURE_EMBEDDED_STORAGE = 'embeddedStorage';
1757
1863
  const PLAN_FEATURE_MESSAGE = 'message';
1758
1864
  const PLAN_FEATURE_TOOL_CALL = 'toolCall';
1865
+ const PLAN_FEATURE_RESOURCE_SYNC = 'resourceSync';
1759
1866
  // Stable code returned on a quota block so clients can branch on it (402).
1760
1867
  const PLAN_LIMIT_ERROR_CODE = 'PLAN_LIMIT_EXCEEDED';
1761
1868
  // Billing meter event names. The metering cron reports per-period OVERAGE
@@ -1923,6 +2030,7 @@ exports.constants = {
1923
2030
  PLAN_FEATURE_EMBEDDED_STORAGE,
1924
2031
  PLAN_FEATURE_MESSAGE,
1925
2032
  PLAN_FEATURE_TOOL_CALL,
2033
+ PLAN_FEATURE_RESOURCE_SYNC,
1926
2034
  PLAN_LIMIT_ERROR_CODE,
1927
2035
  BILLING_METER_MESSAGES,
1928
2036
  BILLING_METER_SHARED_MESSAGES,
@@ -2117,6 +2225,15 @@ exports.constants = {
2117
2225
  RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER,
2118
2226
  RESOURCE_SOURCE_TYPE_CUSTOM_CODE,
2119
2227
  RESOURCE_SOURCE_TYPES,
2228
+ RESOURCE_SYNC_INTERVAL_OFF,
2229
+ RESOURCE_SYNC_INTERVAL_DAILY,
2230
+ RESOURCE_SYNC_INTERVAL_WEEKLY,
2231
+ RESOURCE_SYNC_INTERVALS,
2232
+ RESOURCE_SYNC_INTERVAL_DEFAULT,
2233
+ RESOURCE_SYNC_INTERVAL_MS,
2234
+ RESOURCE_SYNC_BATCH_SIZE,
2235
+ RESOURCE_SYNC_MANUAL_COOLDOWN_MS,
2236
+ RESOURCE_SYNC_STALE_MS,
2120
2237
  CRAWL_RENDERER_CHEERIO,
2121
2238
  CRAWL_RENDERER_PLAYWRIGHT,
2122
2239
  CRAWL_RENDERERS,
@@ -2175,6 +2292,11 @@ exports.constants = {
2175
2292
  SHARED_KEY_HISTORY_LIMIT,
2176
2293
  SHARED_KEY_MAX_TOOL_LOOPS,
2177
2294
  CHANNEL_MAX_TOOLS,
2295
+ TOOL_CONFIRMATION_TTL_MS,
2296
+ TOOL_CONFIRMATION_REQUIRED_RESULT,
2297
+ TOOL_CONFIRMATION_FALLBACK_QUESTION,
2298
+ TOOL_CONFIRMATION_ASK_INSTRUCTION,
2299
+ TOOL_CONFIRMATION_SYSTEM_NOTE,
2178
2300
  CHANNEL_DEBOUNCE_DEFAULT_MS,
2179
2301
  CHANNEL_DEBOUNCE_MIN_MS,
2180
2302
  CHANNEL_DEBOUNCE_MAX_MS,
@@ -2207,8 +2329,10 @@ exports.constants = {
2207
2329
  DISCORD_TIMESTAMP_HEADER,
2208
2330
  DISCORD_INTERACTION_TYPE_PING,
2209
2331
  DISCORD_INTERACTION_TYPE_APPLICATION_COMMAND,
2332
+ DISCORD_INTERACTION_TYPE_MESSAGE_COMPONENT,
2210
2333
  DISCORD_INTERACTION_RESPONSE_PONG,
2211
2334
  DISCORD_INTERACTION_RESPONSE_DEFERRED,
2335
+ DISCORD_INTERACTION_RESPONSE_UPDATE_MESSAGE,
2212
2336
  DISCORD_CHANNEL_TYPE_DM,
2213
2337
  DISCORD_CHANNEL_TYPE_GROUP_DM,
2214
2338
  DISCORD_GATEWAY_OP_DISPATCH,
@@ -2223,6 +2347,7 @@ exports.constants = {
2223
2347
  WHATSAPP_API_BASE,
2224
2348
  WHATSAPP_API_VERSION,
2225
2349
  WHATSAPP_MESSAGE_LIMIT,
2350
+ WHATSAPP_INTERACTIVE_BODY_LIMIT,
2226
2351
  WHATSAPP_SIGNATURE_HEADER,
2227
2352
  WHATSAPP_SIGNATURE_PREFIX,
2228
2353
  WHATSAPP_HUB_MODE_PARAM,
@@ -2238,6 +2363,12 @@ exports.constants = {
2238
2363
  CHUNK_TARGET_CHARS,
2239
2364
  CHUNK_OVERLAP_CHARS,
2240
2365
  EMBED_BATCH_SIZE,
2366
+ RESOURCE_SEARCH_CANDIDATES,
2367
+ RESOURCE_SEARCH_RRF_K,
2368
+ RESOURCE_SEARCH_COMMON_TERM_DIVISOR,
2369
+ RESOURCE_SEARCH_COMMON_TERM_MIN,
2370
+ RESOURCE_SEARCH_RERANK_CANDIDATES,
2371
+ RESOURCE_SEARCH_RERANK_MODEL,
2241
2372
  RESOURCE_HANDLER_SLEEP_AFTER,
2242
2373
  DOCS_URL,
2243
2374
  BASE64_DATA_URI_RE,