@ganju/utils 0.0.8 → 0.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,17 @@ export interface BufferedChannelMessage {
2
2
  text: string;
3
3
  externalMessageId: string | null;
4
4
  receivedAt: number;
5
+ immediate?: boolean;
6
+ command?: BufferedChannelCommand | null;
7
+ reply?: Record<string, unknown> | null;
5
8
  }
9
+ export interface BufferedChannelCommand {
10
+ promptId: string;
11
+ artifactPromptId: string | null;
12
+ promptTitle: string;
13
+ args: Record<string, string>;
14
+ }
15
+ export declare const batchCommand: (messages: BufferedChannelMessage[]) => BufferedChannelCommand | null;
6
16
  export interface ChannelBufferEnvelope {
7
17
  channelId: string;
8
18
  platform: string;
@@ -17,6 +27,7 @@ export interface ChannelBufferEnvelope {
17
27
  export interface ChannelBufferFlush {
18
28
  envelope: ChannelBufferEnvelope;
19
29
  messages: BufferedChannelMessage[];
30
+ batchId: string;
20
31
  }
21
32
  export declare const resolveDebounceMs: (config: unknown) => number;
22
33
  export declare const channelBufferKey: (channelId: string, externalConversationId: string, externalParticipantId: string) => string;
@@ -1 +1 @@
1
- {"version":3,"file":"channelDebounce.d.ts","sourceRoot":"","sources":["../src/channelDebounce.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;CACpB;AAOD,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACpD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAGD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,qBAAqB,CAAC;IAChC,QAAQ,EAAE,sBAAsB,EAAE,CAAC;CACpC;AAOD,eAAO,MAAM,iBAAiB,GAAI,QAAQ,OAAO,KAAG,MAYnD,CAAC;AAKF,eAAO,MAAM,gBAAgB,GAC3B,WAAW,MAAM,EACjB,wBAAwB,MAAM,EAC9B,uBAAuB,MAAM,KAC5B,MAA2E,CAAC;AAI/E,eAAO,MAAM,oBAAoB,GAC/B,UAAU,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,KAChC,MAIuC,CAAC"}
1
+ {"version":3,"file":"channelDebounce.d.ts","sourceRoot":"","sources":["../src/channelDebounce.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IAKnB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,OAAO,CAAC,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAKxC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACxC;AAID,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAID,eAAO,MAAM,YAAY,GACvB,UAAU,sBAAsB,EAAE,KACjC,sBAAsB,GAAG,IAM3B,CAAC;AAOF,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACpD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAGD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,qBAAqB,CAAC;IAChC,QAAQ,EAAE,sBAAsB,EAAE,CAAC;IAGnC,OAAO,EAAE,MAAM,CAAC;CACjB;AAOD,eAAO,MAAM,iBAAiB,GAAI,QAAQ,OAAO,KAAG,MAYnD,CAAC;AAKF,eAAO,MAAM,gBAAgB,GAC3B,WAAW,MAAM,EACjB,wBAAwB,MAAM,EAC9B,uBAAuB,MAAM,KAC5B,MAA2E,CAAC;AAI/E,eAAO,MAAM,oBAAoB,GAC/B,UAAU,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,KAChC,MAIuC,CAAC"}
@@ -1,7 +1,18 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.joinBufferedMessages = exports.channelBufferKey = exports.resolveDebounceMs = void 0;
3
+ exports.joinBufferedMessages = exports.channelBufferKey = exports.resolveDebounceMs = exports.batchCommand = void 0;
4
4
  const constants_1 = require("./constants");
5
+ // The prompt a batch runs: the newest command in it, since that is the one the
6
+ // participant asked for last. Null for a batch of plain text.
7
+ const batchCommand = (messages) => {
8
+ for (let i = messages.length - 1; i >= 0; i--) {
9
+ const command = messages[i].command;
10
+ if (command)
11
+ return command;
12
+ }
13
+ return null;
14
+ };
15
+ exports.batchCommand = batchCommand;
5
16
  // Read the channel's debounce window off its `config`, clamped to the supported
6
17
  // range. An unset value takes the default; an explicit 0 disables buffering, so
7
18
  // the channel answers every message as it arrives. Anything malformed (a string,
@@ -6,6 +6,17 @@ export declare const chunkText: (text: string, options?: {
6
6
  targetChars?: number;
7
7
  overlapChars?: number;
8
8
  }) => string[];
9
+ export type HeadingSyntax = 'markdown' | 'html';
10
+ /**
11
+ * Which heading syntax a document is written in, or null when it has none we
12
+ * can trust. Plain text is left out on purpose: a line that happens to start
13
+ * with "# " there is as likely a comment or a list as a title.
14
+ */
15
+ export declare const headingSyntaxFor: (input: {
16
+ mimeType?: string | null;
17
+ fileName?: string | null;
18
+ isWebPage?: boolean;
19
+ }) => HeadingSyntax | null;
9
20
  export declare const buildHeader: (input: {
10
21
  title?: string | null;
11
22
  description?: string | null;
@@ -13,9 +24,10 @@ export declare const buildHeader: (input: {
13
24
  mimeType?: string | null;
14
25
  fileName?: string | null;
15
26
  }) => string;
16
- export interface ChunkMetadata extends ExtractedDocumentMetadata {
17
- chunkIndexInPage: number;
18
- chunksInPage: number;
27
+ export interface ChunkMetadata extends Partial<ExtractedDocumentMetadata> {
28
+ chunkIndexInPage?: number;
29
+ chunksInPage?: number;
30
+ headingPath?: string[];
19
31
  }
20
32
  export interface PreparedChunk {
21
33
  content: string;
@@ -24,5 +36,7 @@ export interface PreparedChunk {
24
36
  export declare const prepareChunks: (header: string, documents: ExtractedDocument[] | null, fallbackContent: string | null, options?: {
25
37
  targetChars?: number;
26
38
  overlapChars?: number;
39
+ mimeType?: string | null;
40
+ fileName?: string | null;
27
41
  }) => PreparedChunk[];
28
42
  //# sourceMappingURL=chunking.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,KAAK,EACV,iBAAiB,EACjB,yBAAyB,EAC1B,MAAM,qBAAqB,CAAC;AAE7B,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,SAAS,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AASpE,eAAO,MAAM,cAAc,GACzB,MAAM,MAAM,EACZ,QAAQ,MAAM,EACd,aAAY,SAAS,SAAS,EAA+B,KAC5D,MAAM,EA8BR,CAAC;AAEF,eAAO,MAAM,SAAS,GACpB,MAAM,MAAM,EACZ,UAAU;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KACxD,MAAM,EAeR,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO;IACjC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KAAG,MASW,CAAC;AAEhB,MAAM,WAAW,aAAc,SAAQ,yBAAyB;IAC9D,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CAChC;AAED,eAAO,MAAM,aAAa,GACxB,QAAQ,MAAM,EACd,WAAW,iBAAiB,EAAE,GAAG,IAAI,EACrC,iBAAiB,MAAM,GAAG,IAAI,EAC9B,UAAU;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KACxD,aAAa,EAgCf,CAAC"}
1
+ {"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,KAAK,EACV,iBAAiB,EACjB,yBAAyB,EAC1B,MAAM,qBAAqB,CAAC;AAE7B,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,SAAS,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AASpE,eAAO,MAAM,cAAc,GACzB,MAAM,MAAM,EACZ,QAAQ,MAAM,EACd,aAAY,SAAS,SAAS,EAA+B,KAC5D,MAAM,EA8BR,CAAC;AA4BF,eAAO,MAAM,SAAS,GACpB,MAAM,MAAM,EACZ,UAAU;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KACxD,MAAM,EAAoD,CAAC;AAE9D,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,MAAM,CAAC;AAehD;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,GAAI,OAAO;IACtC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,KAAG,aAAa,GAAG,IAUnB,CAAC;AAuEF,eAAO,MAAM,WAAW,GAAI,OAAO;IACjC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KAAG,MASW,CAAC;AAIhB,MAAM,WAAW,aAAc,SAAQ,OAAO,CAAC,yBAAyB,CAAC;IACvE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CAChC;AAID,eAAO,MAAM,aAAa,GACxB,QAAQ,MAAM,EACd,WAAW,iBAAiB,EAAE,GAAG,IAAI,EACrC,iBAAiB,MAAM,GAAG,IAAI,EAC9B,UAAU;IACR,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KACA,aAAa,EA4Cf,CAAC"}
package/dist/chunking.js CHANGED
@@ -1,7 +1,8 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.prepareChunks = exports.buildHeader = exports.chunkText = exports.splitRecursive = void 0;
3
+ exports.prepareChunks = exports.buildHeader = exports.headingSyntaxFor = exports.chunkText = exports.splitRecursive = void 0;
4
4
  const constants_1 = require("./constants");
5
+ const sanitize_1 = require("./sanitize");
5
6
  const trySplit = (text, sep) => {
6
7
  if (typeof sep === 'string') {
7
8
  return text.includes(sep) ? text.split(sep) : null;
@@ -40,23 +41,115 @@ const splitRecursive = (text, target, separators = constants_1.constants.CHUNK_S
40
41
  return chunks;
41
42
  };
42
43
  exports.splitRecursive = splitRecursive;
43
- const chunkText = (text, options) => {
44
+ // Each piece keeps its `raw` text — the slice of the input it covers, without
45
+ // the overlap borrowed from the piece before — so a caller can walk the input
46
+ // in order without reading any of it twice.
47
+ const chunkPieces = (text, options) => {
44
48
  const target = options?.targetChars ?? constants_1.constants.CHUNK_TARGET_CHARS;
45
49
  const overlap = options?.overlapChars ?? constants_1.constants.CHUNK_OVERLAP_CHARS;
46
50
  const trimmed = text.trim();
47
51
  if (!trimmed)
48
52
  return [];
49
53
  if (trimmed.length <= target)
50
- return [trimmed];
54
+ return [{ raw: trimmed, content: trimmed }];
51
55
  const rawChunks = (0, exports.splitRecursive)(trimmed, target);
52
- const withOverlap = [];
56
+ const pieces = [];
53
57
  for (let i = 0; i < rawChunks.length; i++) {
54
58
  const prev = i > 0 ? rawChunks[i - 1].slice(-overlap) : '';
55
- withOverlap.push((prev ? prev + '\n' : '') + rawChunks[i]);
59
+ pieces.push({
60
+ raw: rawChunks[i],
61
+ content: (prev ? prev + '\n' : '') + rawChunks[i]
62
+ });
56
63
  }
57
- return withOverlap.filter(c => c.trim().length > 0);
64
+ return pieces.filter(p => p.content.trim().length > 0);
58
65
  };
66
+ const chunkText = (text, options) => chunkPieces(text, options).map(p => p.content);
59
67
  exports.chunkText = chunkText;
68
+ const MARKDOWN_HEADING = /^ {0,3}(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/;
69
+ const MARKDOWN_FENCE = /^ {0,3}(```|~~~)/;
70
+ const HTML_HEADING = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi;
71
+ const MAX_HEADING_CHARS = 120;
72
+ const MARKDOWN_FILE_NAME = /\.(md|markdown|mdx)$/i;
73
+ /**
74
+ * Which heading syntax a document is written in, or null when it has none we
75
+ * can trust. Plain text is left out on purpose: a line that happens to start
76
+ * with "# " there is as likely a comment or a list as a title.
77
+ */
78
+ const headingSyntaxFor = (input) => {
79
+ // Crawled pages are flattened to text with their headings written back as
80
+ // Markdown lines, whatever the page's own content type was.
81
+ if (input.isWebPage)
82
+ return 'markdown';
83
+ if (input.mimeType === constants_1.constants.MIMETYPE_TEXT_MARKDOWN)
84
+ return 'markdown';
85
+ if (input.mimeType === constants_1.constants.MIMETYPE_TEXT_HTML)
86
+ return 'html';
87
+ if (input.fileName && MARKDOWN_FILE_NAME.test(input.fileName)) {
88
+ return 'markdown';
89
+ }
90
+ return null;
91
+ };
92
+ exports.headingSyntaxFor = headingSyntaxFor;
93
+ const cleanHeading = (text) => text.replace(/\s+/g, ' ').trim().slice(0, MAX_HEADING_CHARS);
94
+ const scanMarkdownHeadings = (text, state) => {
95
+ const events = [];
96
+ let offset = 0;
97
+ for (const line of text.split('\n')) {
98
+ // A `#` inside a fenced block is a shell comment or a preprocessor line,
99
+ // not a title — and a fence can open in one chunk and close in the next.
100
+ if (MARKDOWN_FENCE.test(line)) {
101
+ state.inFence = !state.inFence;
102
+ }
103
+ else if (!state.inFence) {
104
+ const match = MARKDOWN_HEADING.exec(line);
105
+ const heading = match ? cleanHeading(match[2]) : '';
106
+ if (match && heading) {
107
+ events.push({ offset, level: match[1].length, text: heading });
108
+ }
109
+ }
110
+ offset += line.length + 1;
111
+ }
112
+ return events;
113
+ };
114
+ const scanHtmlHeadings = (text) => [...text.matchAll(HTML_HEADING)]
115
+ .map(match => ({
116
+ offset: match.index ?? 0,
117
+ level: Number(match[1]),
118
+ text: cleanHeading((0, sanitize_1.decodeEntities)(match[2].replace(/<[^>]*>/g, ' ')))
119
+ }))
120
+ .filter(event => event.text);
121
+ /**
122
+ * Follows the heading hierarchy across a document's chunks, fed in order.
123
+ * Returns, for each chunk, the path of headings it sits under — the section a
124
+ * reader would be pointed to. A chunk that opens with a heading belongs to that
125
+ * heading; one that starts mid-section belongs to the section it continues,
126
+ * even when a later heading begins inside it.
127
+ */
128
+ const createHeadingTracker = (syntax) => {
129
+ let stack = [];
130
+ const fence = { inFence: false };
131
+ const apply = (event) => {
132
+ stack = stack.filter(h => h.level < event.level);
133
+ stack.push({ level: event.level, text: event.text });
134
+ };
135
+ return (raw) => {
136
+ const events = syntax === 'markdown'
137
+ ? scanMarkdownHeadings(raw, fence)
138
+ : scanHtmlHeadings(raw);
139
+ const [first, ...rest] = events;
140
+ // Before a document's first heading there is no section to continue — only
141
+ // the metadata header, or the markup around an HTML body — so a chunk that
142
+ // holds the first heading belongs to it wherever in the chunk it falls.
143
+ const opensWithHeading = first !== undefined &&
144
+ (stack.length === 0 || raw.slice(0, first.offset).trim() === '');
145
+ if (opensWithHeading)
146
+ apply(first);
147
+ const path = stack.map(h => h.text);
148
+ for (const event of opensWithHeading ? rest : events)
149
+ apply(event);
150
+ return path;
151
+ };
152
+ };
60
153
  const buildHeader = (input) => [
61
154
  input.title ? `Title: ${input.title}` : null,
62
155
  input.description ? `Description: ${input.description}` : null,
@@ -67,37 +160,50 @@ const buildHeader = (input) => [
67
160
  .filter(Boolean)
68
161
  .join('\n');
69
162
  exports.buildHeader = buildHeader;
163
+ const CONTROL_CHARS = /[\x00-\x08\x0B\x0C\x0E-\x1F]/g;
70
164
  const prepareChunks = (header, documents, fallbackContent, options) => {
71
165
  if (documents && documents.length > 0) {
72
166
  const out = [];
73
167
  documents.forEach((doc, docIndex) => {
74
- const body = (doc.pageContent || '')
75
- .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
76
- .trim();
168
+ const body = (doc.pageContent || '').replace(CONTROL_CHARS, '').trim();
77
169
  if (!body)
78
170
  return;
79
171
  const text = docIndex === 0 && header ? `${header}\n\n${body}` : body;
80
- const pieces = (0, exports.chunkText)(text, options);
172
+ const syntax = (0, exports.headingSyntaxFor)({
173
+ mimeType: doc.metadata?.source?.mimeType ?? options?.mimeType,
174
+ fileName: doc.metadata?.source?.fileName ?? options?.fileName,
175
+ isWebPage: !!doc.metadata?.web
176
+ });
177
+ const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
178
+ const pieces = chunkPieces(text, options);
81
179
  pieces.forEach((piece, pieceIndex) => {
180
+ const headingPath = headingPathOf?.(piece.raw) ?? [];
82
181
  out.push({
83
- content: piece,
182
+ content: piece.content,
84
183
  metadata: {
85
184
  ...doc.metadata,
86
185
  chunkIndexInPage: pieceIndex,
87
- chunksInPage: pieces.length
186
+ chunksInPage: pieces.length,
187
+ ...(headingPath.length > 0 ? { headingPath } : {})
88
188
  }
89
189
  });
90
190
  });
91
191
  });
92
192
  return out;
93
193
  }
94
- const body = (fallbackContent || '')
95
- .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
96
- .trim();
194
+ const body = (fallbackContent || '').replace(CONTROL_CHARS, '').trim();
97
195
  const fullText = [header, body].filter(Boolean).join('\n\n');
98
- return (0, exports.chunkText)(fullText, options).map(content => ({
99
- content,
100
- metadata: null
101
- }));
196
+ const syntax = (0, exports.headingSyntaxFor)({
197
+ mimeType: options?.mimeType,
198
+ fileName: options?.fileName
199
+ });
200
+ const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
201
+ return chunkPieces(fullText, options).map(piece => {
202
+ const headingPath = headingPathOf?.(piece.raw) ?? [];
203
+ return {
204
+ content: piece.content,
205
+ metadata: headingPath.length > 0 ? { headingPath } : null
206
+ };
207
+ });
102
208
  };
103
209
  exports.prepareChunks = prepareChunks;
@@ -6,6 +6,8 @@ declare const ONE_DRIVE_TAB_MY_FILES: "my-files";
6
6
  declare const ONE_DRIVE_TAB_SHARED_WITH_ME: "shared-with-me";
7
7
  declare const ONE_DRIVE_TAB_RECENT: "recent";
8
8
  declare const ONE_DRIVE_TAB_DRIVES: "drives";
9
+ declare const RESOURCE_SYNC_INTERVAL_DAILY: "daily";
10
+ declare const RESOURCE_SYNC_INTERVAL_WEEKLY: "weekly";
9
11
  declare const LLM_PROVIDER_ANTHROPIC: "anthropic";
10
12
  declare const LLM_PROVIDER_OPENAI: "openai";
11
13
  declare const LLM_PROVIDER_GOOGLE: "google";
@@ -59,6 +61,7 @@ interface PlanLimits {
59
61
  toolCallHardCap: number | null;
60
62
  includedMessages: number;
61
63
  includedEmbeddedBytes: number;
64
+ autoSyncIntervals: ReadonlyArray<typeof RESOURCE_SYNC_INTERVAL_DAILY | typeof RESOURCE_SYNC_INTERVAL_WEEKLY>;
62
65
  }
63
66
  export type { PlanLimits };
64
67
  export declare const constants: {
@@ -104,6 +107,7 @@ export declare const constants: {
104
107
  PLAN_FEATURE_EMBEDDED_STORAGE: "embeddedStorage";
105
108
  PLAN_FEATURE_MESSAGE: "message";
106
109
  PLAN_FEATURE_TOOL_CALL: "toolCall";
110
+ PLAN_FEATURE_RESOURCE_SYNC: "resourceSync";
107
111
  PLAN_LIMIT_ERROR_CODE: string;
108
112
  BILLING_METER_MESSAGES: string;
109
113
  BILLING_METER_SHARED_MESSAGES: string;
@@ -309,6 +313,17 @@ export declare const constants: {
309
313
  RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER: "ONE_DRIVE_FOLDER";
310
314
  RESOURCE_SOURCE_TYPE_CUSTOM_CODE: "CUSTOM_CODE";
311
315
  RESOURCE_SOURCE_TYPES: ("FILE" | "WEBSITE" | "GOOGLE_DRIVE_FOLDER" | "ONE_DRIVE_FOLDER" | "CUSTOM_CODE")[];
316
+ RESOURCE_SYNC_INTERVAL_OFF: "off";
317
+ RESOURCE_SYNC_INTERVAL_DAILY: "daily";
318
+ RESOURCE_SYNC_INTERVAL_WEEKLY: "weekly";
319
+ RESOURCE_SYNC_INTERVALS: readonly ["off", "daily", "weekly"];
320
+ RESOURCE_SYNC_INTERVAL_DEFAULT: "weekly";
321
+ RESOURCE_SYNC_INTERVAL_MS: {
322
+ readonly daily: number;
323
+ readonly weekly: number;
324
+ };
325
+ RESOURCE_SYNC_BATCH_SIZE: number;
326
+ RESOURCE_SYNC_MANUAL_COOLDOWN_MS: number;
312
327
  CRAWL_RENDERER_CHEERIO: "cheerio";
313
328
  CRAWL_RENDERER_PLAYWRIGHT: "playwright";
314
329
  CRAWL_RENDERERS: ("cheerio" | "playwright")[];
@@ -362,6 +377,9 @@ export declare const constants: {
362
377
  model: string;
363
378
  label: string;
364
379
  }[];
380
+ LLM_CATALOG_CUSTOM_KEY: string;
381
+ LLM_BASE_URL_REQUIRED_MESSAGE: string;
382
+ LLM_BASE_URL_INVALID_MESSAGE: string;
365
383
  LLM_SYSTEM_DEFAULT: string;
366
384
  MAX_TOOL_LOOPS: number;
367
385
  CHANNEL_HISTORY_LIMIT: number;
@@ -431,6 +449,12 @@ export declare const constants: {
431
449
  CHUNK_TARGET_CHARS: number;
432
450
  CHUNK_OVERLAP_CHARS: number;
433
451
  EMBED_BATCH_SIZE: number;
452
+ RESOURCE_SEARCH_CANDIDATES: number;
453
+ RESOURCE_SEARCH_RRF_K: number;
454
+ RESOURCE_SEARCH_COMMON_TERM_DIVISOR: number;
455
+ RESOURCE_SEARCH_COMMON_TERM_MIN: number;
456
+ RESOURCE_SEARCH_RERANK_CANDIDATES: number;
457
+ RESOURCE_SEARCH_RERANK_MODEL: string;
434
458
  RESOURCE_HANDLER_SLEEP_AFTER: string;
435
459
  DOCS_URL: string;
436
460
  BASE64_DATA_URI_RE: RegExp;
@@ -1 +1 @@
1
- {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AA6JA,QAAA,MAAM,yBAAyB,EAAiB,UAAU,CAAC;AAC3D,QAAA,MAAM,+BAA+B,EAAuB,gBAAgB,CAAC;AAC7E,QAAA,MAAM,8BAA8B,EAAsB,eAAe,CAAC;AAC1E,QAAA,MAAM,wBAAwB,EAAgB,SAAS,CAAC;AAuCxD,QAAA,MAAM,sBAAsB,EAAiB,UAAU,CAAC;AACxD,QAAA,MAAM,4BAA4B,EAAuB,gBAAgB,CAAC;AAC1E,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAClD,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAmYlD,QAAA,MAAM,sBAAsB,EAAkB,WAAW,CAAC;AAC1D,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAGjD,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAkTjD,MAAM,MAAM,mBAAmB,GAC3B;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC9D;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA2+BpE,UAAU,UAAU;IAElB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAGhC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAMjC,sBAAsB,EAAE,MAAM,CAAC;IAM/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,SAAS,EAAE,OAAO,CAAC;IAKnB,eAAe,EAAE,OAAO,CAAC;IAOzB,gBAAgB,EAAE,OAAO,CAAC;IAK1B,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAQ3C,iBAAiB,EAAE,MAAM,CAAC;IAM1B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAG/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAiPD,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAjgEhB,OAAO,yBAAyB,GAChC,OAAO,+BAA+B,GACtC,OAAO,8BAA8B,GACrC,OAAO,wBAAwB;eAC5B,MAAM;;;;;;;;;;;;;;;;;;eAsCT,OAAO,sBAAsB,GAC7B,OAAO,4BAA4B,GACnC,OAAO,oBAAoB,GAC3B,OAAO,oBAAoB;eACxB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA6YT,OAAO,mBAAmB,GAC1B,OAAO,mBAAmB,GAC1B,OAAO,sBAAsB;eAC1B,MAAM;eACN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eA6NwB,MAAM,GAAG,MAAM;cAAQ,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAy6DnE,CAAC"}
1
+ {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AA6JA,QAAA,MAAM,yBAAyB,EAAiB,UAAU,CAAC;AAC3D,QAAA,MAAM,+BAA+B,EAAuB,gBAAgB,CAAC;AAC7E,QAAA,MAAM,8BAA8B,EAAsB,eAAe,CAAC;AAC1E,QAAA,MAAM,wBAAwB,EAAgB,SAAS,CAAC;AAuCxD,QAAA,MAAM,sBAAsB,EAAiB,UAAU,CAAC;AACxD,QAAA,MAAM,4BAA4B,EAAuB,gBAAgB,CAAC;AAC1E,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAClD,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAoFlD,QAAA,MAAM,4BAA4B,EAAc,OAAO,CAAC;AACxD,QAAA,MAAM,6BAA6B,EAAe,QAAQ,CAAC;AAyU3D,QAAA,MAAM,sBAAsB,EAAkB,WAAW,CAAC;AAC1D,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAGjD,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAwVjD,MAAM,MAAM,mBAAmB,GAC3B;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC9D;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA0+BpE,UAAU,UAAU;IAElB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAGhC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAMjC,sBAAsB,EAAE,MAAM,CAAC;IAM/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,SAAS,EAAE,OAAO,CAAC;IAKnB,eAAe,EAAE,OAAO,CAAC;IAOzB,gBAAgB,EAAE,OAAO,CAAC;IAK1B,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAQ3C,iBAAiB,EAAE,MAAM,CAAC;IAM1B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAG/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAI9B,iBAAiB,EAAE,aAAa,CAC9B,OAAO,4BAA4B,GAAG,OAAO,6BAA6B,CAC3E,CAAC;CACH;AA2PD,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAjlEhB,OAAO,yBAAyB,GAChC,OAAO,+BAA+B,GACtC,OAAO,8BAA8B,GACrC,OAAO,wBAAwB;eAC5B,MAAM;;;;;;;;;;;;;;;;;;eAsCT,OAAO,sBAAsB,GAC7B,OAAO,4BAA4B,GACnC,OAAO,oBAAoB,GAC3B,OAAO,oBAAoB;eACxB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAwaT,OAAO,mBAAmB,GAC1B,OAAO,mBAAmB,GAC1B,OAAO,sBAAsB;eAC1B,MAAM;eACN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAmQwB,MAAM,GAAG,MAAM;cAAQ,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA08DnE,CAAC"}
package/dist/constants.js CHANGED
@@ -208,6 +208,32 @@ const RESOURCE_SOURCE_TYPES = [
208
208
  RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER,
209
209
  RESOURCE_SOURCE_TYPE_CUSTOM_CODE
210
210
  ];
211
+ // How often a source that lives somewhere else — a crawled website, a Drive or
212
+ // OneDrive folder, a file imported from either — is refreshed from it. Every
213
+ // source is stored with an interval, weekly unless its owner changes it; the
214
+ // hourly sync job only acts on it for a plan that includes automatic sync, so a
215
+ // Free org's sources start syncing the moment it upgrades.
216
+ const RESOURCE_SYNC_INTERVAL_OFF = 'off';
217
+ const RESOURCE_SYNC_INTERVAL_DAILY = 'daily';
218
+ const RESOURCE_SYNC_INTERVAL_WEEKLY = 'weekly';
219
+ const RESOURCE_SYNC_INTERVALS = [
220
+ RESOURCE_SYNC_INTERVAL_OFF,
221
+ RESOURCE_SYNC_INTERVAL_DAILY,
222
+ RESOURCE_SYNC_INTERVAL_WEEKLY
223
+ ];
224
+ const RESOURCE_SYNC_INTERVAL_DEFAULT = RESOURCE_SYNC_INTERVAL_WEEKLY;
225
+ const RESOURCE_SYNC_INTERVAL_MS = {
226
+ [RESOURCE_SYNC_INTERVAL_DAILY]: 24 * 60 * 60 * 1000,
227
+ [RESOURCE_SYNC_INTERVAL_WEEKLY]: 7 * 24 * 60 * 60 * 1000
228
+ };
229
+ // Sources started per hourly run. A sync is cheap to start and expensive to
230
+ // run — a website re-crawls every page — so the job works through a backlog
231
+ // over several hours rather than launching all of it at once.
232
+ const RESOURCE_SYNC_BATCH_SIZE = 50;
233
+ // How soon a source can be synced by hand again. Manual sync is open to every
234
+ // plan, and a website sync re-crawls every page, so it can't be a button to
235
+ // press in a loop.
236
+ const RESOURCE_SYNC_MANUAL_COOLDOWN_MS = 5 * 60 * 1000;
211
237
  const CRAWL_RENDERER_CHEERIO = 'cheerio';
212
238
  const CRAWL_RENDERER_PLAYWRIGHT = 'playwright';
213
239
  const CRAWL_RENDERERS = [CRAWL_RENDERER_CHEERIO, CRAWL_RENDERER_PLAYWRIGHT];
@@ -496,8 +522,8 @@ const DEFAULT_LLM_SYSTEM_PROMPT = 'You are a helpful assistant. Answer the user
496
522
  const LLM_CATALOG = [
497
523
  {
498
524
  provider: LLM_PROVIDER_GOOGLE,
499
- model: 'gemini-3-flash-preview',
500
- label: 'Gemini 3 Flash'
525
+ model: 'gemini-3.8-flash',
526
+ label: 'Gemini 3.8 Flash'
501
527
  },
502
528
  {
503
529
  provider: LLM_PROVIDER_GOOGLE,
@@ -506,13 +532,13 @@ const LLM_CATALOG = [
506
532
  },
507
533
  {
508
534
  provider: LLM_PROVIDER_OPENAI,
509
- model: 'gpt-4o-mini',
510
- label: 'GPT-4o mini'
535
+ model: 'gpt-5.4-mini',
536
+ label: 'GPT-5.4 mini'
511
537
  },
512
538
  {
513
539
  provider: LLM_PROVIDER_OPENAI,
514
- model: 'gpt-4o',
515
- label: 'GPT-4o'
540
+ model: 'gpt-5.5',
541
+ label: 'GPT-5.5'
516
542
  },
517
543
  {
518
544
  provider: LLM_PROVIDER_ANTHROPIC,
@@ -521,15 +547,30 @@ const LLM_CATALOG = [
521
547
  },
522
548
  {
523
549
  provider: LLM_PROVIDER_ANTHROPIC,
524
- model: 'claude-sonnet-4-6',
525
- label: 'Claude Sonnet 4.6'
550
+ model: 'claude-sonnet-5',
551
+ label: 'Claude Sonnet 5'
526
552
  },
527
553
  {
528
554
  provider: LLM_PROVIDER_ANTHROPIC,
529
- model: 'claude-opus-4.7',
530
- label: 'Claude Sonnet 4.7'
555
+ model: 'claude-opus-5',
556
+ label: 'Claude Opus 5'
531
557
  }
532
558
  ];
559
+ // The picker's escape hatch. Anything that speaks the OpenAI chat-completions
560
+ // protocol — a gateway, an aggregator, a self-hosted server — is reachable
561
+ // without a catalog row: the customer supplies the model id and the base URL,
562
+ // and the request goes out through the OpenAI adapter. Not a provider value, so
563
+ // it can never reach the database; the dashboard translates it into
564
+ // LLM_PROVIDER_OPENAI_COMPATIBLE on submit.
565
+ const LLM_CATALOG_CUSTOM_KEY = 'custom';
566
+ // Validation messages for an organization LLM's base URL. Constants rather than
567
+ // inline strings because localizeZodIssue keys its translations on the exact
568
+ // English text, so the two files have to name the same value. Both also carry a
569
+ // word matchStatus recognises ("required", "must be"): the update path re-checks
570
+ // the merged row and throws the message as a plain Error, and anything
571
+ // unrecognised there becomes an opaque 500 instead of a 400.
572
+ const LLM_BASE_URL_REQUIRED_MESSAGE = 'A base URL is required for an OpenAI-compatible model';
573
+ const LLM_BASE_URL_INVALID_MESSAGE = 'Base URL must be a public https:// address';
533
574
  const LLM_SYSTEM_DEFAULT = 'SYSTEM_DEFAULT';
534
575
  const MAX_TOOL_LOOPS = 8;
535
576
  // How many recent messages a channel turn replays as context.
@@ -692,6 +733,24 @@ const EMBEDDING_DIMENSIONS = 1536;
692
733
  const CHUNK_TARGET_CHARS = 2000;
693
734
  const CHUNK_OVERLAP_CHARS = 200;
694
735
  const EMBED_BATCH_SIZE = 96;
736
+ // Hybrid resource search. Each ranker contributes its top candidates, and the
737
+ // two lists are merged with reciprocal rank fusion: score = sum of 1/(k + rank).
738
+ // Fusion reads ranks, never raw scores, so a cosine distance and a ts_rank never
739
+ // have to be put on one scale. k = 60 is the constant from the original RRF
740
+ // paper; it keeps one list's #1 from outweighing a chunk both lists agree on.
741
+ const RESOURCE_SEARCH_CANDIDATES = 50;
742
+ const RESOURCE_SEARCH_RRF_K = 60;
743
+ // A query word found in more than 1 of every 10 chunks of the artifact is
744
+ // dropped from the lexical query — never below 5 chunks, so a small artifact
745
+ // keeps its words.
746
+ const RESOURCE_SEARCH_COMMON_TERM_DIVISOR = 10;
747
+ const RESOURCE_SEARCH_COMMON_TERM_MIN = 5;
748
+ // Second stage, on wherever the Worker has an AI binding: the top fused
749
+ // candidates are re-scored by a cross-encoder, which reads query and chunk
750
+ // together instead of comparing two independently computed vectors. One model
751
+ // call per search; a failure falls back to the fused order.
752
+ const RESOURCE_SEARCH_RERANK_CANDIDATES = 30;
753
+ const RESOURCE_SEARCH_RERANK_MODEL = '@cf/baai/bge-reranker-base';
695
754
  const RESOURCE_HANDLER_SLEEP_AFTER = '10m';
696
755
  const DOCS_URL = 'https://docs.ganju.ai';
697
756
  const BASE64_DATA_URI_RE = /data:image\/[a-zA-Z0-9+\-.]+;base64,[A-Za-z0-9+/=\s]+/g;
@@ -996,8 +1055,7 @@ const MCP_PROXY_TOOL_NAME_MAX = 64;
996
1055
  //
997
1056
  // The row's config holds only `activeVersionId` plus the egress/limit settings;
998
1057
  // names and schemas live on the version. That's what keeps MCP boot off the
999
- // dispatcher — a slow or broken script must never break tools/list. See
1000
- // docs/CUSTOM_TOOLS.md ("Boot contract").
1058
+ // dispatcher — a slow or broken script must never break tools/list.
1001
1059
  const TOOL_DEFINITION_KEY_CUSTOM_CODE = 'custom-code';
1002
1060
  // A version is uploaded as `draft`, becomes `published` when it is made active,
1003
1061
  // and is `archived` when a later version replaces it. Exactly one version per
@@ -1677,7 +1735,8 @@ const PLAN_LIMITS = {
1677
1735
  includedToolCalls: 0,
1678
1736
  toolCallHardCap: 10_000,
1679
1737
  includedMessages: 100,
1680
- includedEmbeddedBytes: 5 * MB
1738
+ includedEmbeddedBytes: 5 * MB,
1739
+ autoSyncIntervals: []
1681
1740
  },
1682
1741
  PRO: {
1683
1742
  maxProjects: null,
@@ -1700,7 +1759,11 @@ const PLAN_LIMITS = {
1700
1759
  includedToolCalls: PRICING_INCLUDED_TOOL_CALLS,
1701
1760
  toolCallHardCap: PRICING_TOOL_CALL_HARD_CAP,
1702
1761
  includedMessages: PRICING_INCLUDED_MESSAGES,
1703
- includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
1762
+ includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
1763
+ autoSyncIntervals: [
1764
+ RESOURCE_SYNC_INTERVAL_DAILY,
1765
+ RESOURCE_SYNC_INTERVAL_WEEKLY
1766
+ ]
1704
1767
  },
1705
1768
  ENTERPRISE: {
1706
1769
  maxProjects: null,
@@ -1723,7 +1786,11 @@ const PLAN_LIMITS = {
1723
1786
  // contract; don't make one plan's abuse limit another plan's product limit.
1724
1787
  toolCallHardCap: null,
1725
1788
  includedMessages: PRICING_INCLUDED_MESSAGES,
1726
- includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
1789
+ includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
1790
+ autoSyncIntervals: [
1791
+ RESOURCE_SYNC_INTERVAL_DAILY,
1792
+ RESOURCE_SYNC_INTERVAL_WEEKLY
1793
+ ]
1727
1794
  }
1728
1795
  };
1729
1796
  // Quota features — used as the `feature` discriminator on PlanLimitError so the
@@ -1741,6 +1808,7 @@ const PLAN_FEATURE_RAW_STORAGE = 'rawStorage';
1741
1808
  const PLAN_FEATURE_EMBEDDED_STORAGE = 'embeddedStorage';
1742
1809
  const PLAN_FEATURE_MESSAGE = 'message';
1743
1810
  const PLAN_FEATURE_TOOL_CALL = 'toolCall';
1811
+ const PLAN_FEATURE_RESOURCE_SYNC = 'resourceSync';
1744
1812
  // Stable code returned on a quota block so clients can branch on it (402).
1745
1813
  const PLAN_LIMIT_ERROR_CODE = 'PLAN_LIMIT_EXCEEDED';
1746
1814
  // Billing meter event names. The metering cron reports per-period OVERAGE
@@ -1783,7 +1851,7 @@ const POLAR_SERVER_PRODUCTION = 'production';
1783
1851
  const CONSENT_DOCUMENT_TERMS = 'terms';
1784
1852
  const CONSENT_DOCUMENT_PRIVACY = 'privacy';
1785
1853
  const CONSENT_DOCUMENTS = [CONSENT_DOCUMENT_TERMS, CONSENT_DOCUMENT_PRIVACY];
1786
- const CONSENT_CURRENT_VERSION = '2026-08-31';
1854
+ const CONSENT_CURRENT_VERSION = '2026-09-10';
1787
1855
  const CONSENT_SOURCE_SIGNUP = 'signup';
1788
1856
  const CONSENT_SOURCE_REACCEPT = 'reaccept';
1789
1857
  const CONSENT_SOURCES = [CONSENT_SOURCE_SIGNUP, CONSENT_SOURCE_REACCEPT];
@@ -1908,6 +1976,7 @@ exports.constants = {
1908
1976
  PLAN_FEATURE_EMBEDDED_STORAGE,
1909
1977
  PLAN_FEATURE_MESSAGE,
1910
1978
  PLAN_FEATURE_TOOL_CALL,
1979
+ PLAN_FEATURE_RESOURCE_SYNC,
1911
1980
  PLAN_LIMIT_ERROR_CODE,
1912
1981
  BILLING_METER_MESSAGES,
1913
1982
  BILLING_METER_SHARED_MESSAGES,
@@ -2102,6 +2171,14 @@ exports.constants = {
2102
2171
  RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER,
2103
2172
  RESOURCE_SOURCE_TYPE_CUSTOM_CODE,
2104
2173
  RESOURCE_SOURCE_TYPES,
2174
+ RESOURCE_SYNC_INTERVAL_OFF,
2175
+ RESOURCE_SYNC_INTERVAL_DAILY,
2176
+ RESOURCE_SYNC_INTERVAL_WEEKLY,
2177
+ RESOURCE_SYNC_INTERVALS,
2178
+ RESOURCE_SYNC_INTERVAL_DEFAULT,
2179
+ RESOURCE_SYNC_INTERVAL_MS,
2180
+ RESOURCE_SYNC_BATCH_SIZE,
2181
+ RESOURCE_SYNC_MANUAL_COOLDOWN_MS,
2105
2182
  CRAWL_RENDERER_CHEERIO,
2106
2183
  CRAWL_RENDERER_PLAYWRIGHT,
2107
2184
  CRAWL_RENDERERS,
@@ -2151,6 +2228,9 @@ exports.constants = {
2151
2228
  DEFAULT_LLM_MODEL,
2152
2229
  DEFAULT_LLM_SYSTEM_PROMPT,
2153
2230
  LLM_CATALOG,
2231
+ LLM_CATALOG_CUSTOM_KEY,
2232
+ LLM_BASE_URL_REQUIRED_MESSAGE,
2233
+ LLM_BASE_URL_INVALID_MESSAGE,
2154
2234
  LLM_SYSTEM_DEFAULT,
2155
2235
  MAX_TOOL_LOOPS,
2156
2236
  CHANNEL_HISTORY_LIMIT,
@@ -2220,6 +2300,12 @@ exports.constants = {
2220
2300
  CHUNK_TARGET_CHARS,
2221
2301
  CHUNK_OVERLAP_CHARS,
2222
2302
  EMBED_BATCH_SIZE,
2303
+ RESOURCE_SEARCH_CANDIDATES,
2304
+ RESOURCE_SEARCH_RRF_K,
2305
+ RESOURCE_SEARCH_COMMON_TERM_DIVISOR,
2306
+ RESOURCE_SEARCH_COMMON_TERM_MIN,
2307
+ RESOURCE_SEARCH_RERANK_CANDIDATES,
2308
+ RESOURCE_SEARCH_RERANK_MODEL,
2223
2309
  RESOURCE_HANDLER_SLEEP_AFTER,
2224
2310
  DOCS_URL,
2225
2311
  BASE64_DATA_URI_RE,