@ganju/utils 0.0.8 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channelDebounce.d.ts +11 -0
- package/dist/channelDebounce.d.ts.map +1 -1
- package/dist/channelDebounce.js +12 -1
- package/dist/chunking.d.ts +17 -3
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +125 -19
- package/dist/constants.d.ts +24 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +102 -16
- package/dist/index.d.ts +76 -27
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -0
- package/dist/llmBaseUrl.d.ts +39 -0
- package/dist/llmBaseUrl.d.ts.map +1 -0
- package/dist/llmBaseUrl.js +65 -0
- package/dist/localizeZodIssue.d.ts.map +1 -1
- package/dist/localizeZodIssue.js +6 -0
- package/dist/resourceRerank.d.ts +20 -0
- package/dist/resourceRerank.d.ts.map +1 -0
- package/dist/resourceRerank.js +37 -0
- package/dist/schema.d.ts +41 -24
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +47 -8
- package/dist/syncableResource.d.ts +9 -0
- package/dist/syncableResource.d.ts.map +1 -0
- package/dist/syncableResource.js +31 -0
- package/dist/toolCatalog.d.ts +1 -1
- package/dist/toolCatalog.js +1 -1
- package/package.json +1 -1
- package/src/channelDebounce.ts +36 -0
- package/src/chunking.ts +166 -23
- package/src/constants.ts +114 -16
- package/src/index.ts +26 -1
- package/src/llmBaseUrl.ts +65 -0
- package/src/localizeZodIssue.ts +6 -0
- package/src/resourceRerank.ts +54 -0
- package/src/schema.ts +80 -11
- package/src/syncableResource.ts +48 -0
- package/src/toolCatalog.ts +1 -1
package/src/constants.ts
CHANGED
|
@@ -278,6 +278,33 @@ const RESOURCE_SOURCE_TYPES = [
|
|
|
278
278
|
RESOURCE_SOURCE_TYPE_CUSTOM_CODE
|
|
279
279
|
];
|
|
280
280
|
|
|
281
|
+
// How often a source that lives somewhere else — a crawled website, a Drive or
|
|
282
|
+
// OneDrive folder, a file imported from either — is refreshed from it. Every
|
|
283
|
+
// source is stored with an interval, weekly unless its owner changes it; the
|
|
284
|
+
// hourly sync job only acts on it for a plan that includes automatic sync, so a
|
|
285
|
+
// Free org's sources start syncing the moment it upgrades.
|
|
286
|
+
const RESOURCE_SYNC_INTERVAL_OFF = 'off' as 'off';
|
|
287
|
+
const RESOURCE_SYNC_INTERVAL_DAILY = 'daily' as 'daily';
|
|
288
|
+
const RESOURCE_SYNC_INTERVAL_WEEKLY = 'weekly' as 'weekly';
|
|
289
|
+
const RESOURCE_SYNC_INTERVALS = [
|
|
290
|
+
RESOURCE_SYNC_INTERVAL_OFF,
|
|
291
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
292
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
293
|
+
] as const;
|
|
294
|
+
const RESOURCE_SYNC_INTERVAL_DEFAULT = RESOURCE_SYNC_INTERVAL_WEEKLY;
|
|
295
|
+
const RESOURCE_SYNC_INTERVAL_MS = {
|
|
296
|
+
[RESOURCE_SYNC_INTERVAL_DAILY]: 24 * 60 * 60 * 1000,
|
|
297
|
+
[RESOURCE_SYNC_INTERVAL_WEEKLY]: 7 * 24 * 60 * 60 * 1000
|
|
298
|
+
} as const;
|
|
299
|
+
// Sources started per hourly run. A sync is cheap to start and expensive to
|
|
300
|
+
// run — a website re-crawls every page — so the job works through a backlog
|
|
301
|
+
// over several hours rather than launching all of it at once.
|
|
302
|
+
const RESOURCE_SYNC_BATCH_SIZE = 50;
|
|
303
|
+
// How soon a source can be synced by hand again. Manual sync is open to every
|
|
304
|
+
// plan, and a website sync re-crawls every page, so it can't be a button to
|
|
305
|
+
// press in a loop.
|
|
306
|
+
const RESOURCE_SYNC_MANUAL_COOLDOWN_MS = 5 * 60 * 1000;
|
|
307
|
+
|
|
281
308
|
const CRAWL_RENDERER_CHEERIO = 'cheerio' as 'cheerio';
|
|
282
309
|
const CRAWL_RENDERER_PLAYWRIGHT = 'playwright' as 'playwright';
|
|
283
310
|
const CRAWL_RENDERERS = [CRAWL_RENDERER_CHEERIO, CRAWL_RENDERER_PLAYWRIGHT];
|
|
@@ -622,8 +649,8 @@ const LLM_CATALOG: ReadonlyArray<{
|
|
|
622
649
|
}> = [
|
|
623
650
|
{
|
|
624
651
|
provider: LLM_PROVIDER_GOOGLE,
|
|
625
|
-
model: 'gemini-3-flash
|
|
626
|
-
label: 'Gemini 3 Flash'
|
|
652
|
+
model: 'gemini-3.8-flash',
|
|
653
|
+
label: 'Gemini 3.8 Flash'
|
|
627
654
|
},
|
|
628
655
|
{
|
|
629
656
|
provider: LLM_PROVIDER_GOOGLE,
|
|
@@ -632,13 +659,13 @@ const LLM_CATALOG: ReadonlyArray<{
|
|
|
632
659
|
},
|
|
633
660
|
{
|
|
634
661
|
provider: LLM_PROVIDER_OPENAI,
|
|
635
|
-
model: 'gpt-
|
|
636
|
-
label: 'GPT-
|
|
662
|
+
model: 'gpt-5.4-mini',
|
|
663
|
+
label: 'GPT-5.4 mini'
|
|
637
664
|
},
|
|
638
665
|
{
|
|
639
666
|
provider: LLM_PROVIDER_OPENAI,
|
|
640
|
-
model: 'gpt-
|
|
641
|
-
label: 'GPT-
|
|
667
|
+
model: 'gpt-5.5',
|
|
668
|
+
label: 'GPT-5.5'
|
|
642
669
|
},
|
|
643
670
|
{
|
|
644
671
|
provider: LLM_PROVIDER_ANTHROPIC,
|
|
@@ -647,16 +674,35 @@ const LLM_CATALOG: ReadonlyArray<{
|
|
|
647
674
|
},
|
|
648
675
|
{
|
|
649
676
|
provider: LLM_PROVIDER_ANTHROPIC,
|
|
650
|
-
model: 'claude-sonnet-
|
|
651
|
-
label: 'Claude Sonnet
|
|
677
|
+
model: 'claude-sonnet-5',
|
|
678
|
+
label: 'Claude Sonnet 5'
|
|
652
679
|
},
|
|
653
680
|
{
|
|
654
681
|
provider: LLM_PROVIDER_ANTHROPIC,
|
|
655
|
-
model: 'claude-opus-
|
|
656
|
-
label: 'Claude
|
|
682
|
+
model: 'claude-opus-5',
|
|
683
|
+
label: 'Claude Opus 5'
|
|
657
684
|
}
|
|
658
685
|
];
|
|
659
686
|
|
|
687
|
+
// The picker's escape hatch. Anything that speaks the OpenAI chat-completions
|
|
688
|
+
// protocol — a gateway, an aggregator, a self-hosted server — is reachable
|
|
689
|
+
// without a catalog row: the customer supplies the model id and the base URL,
|
|
690
|
+
// and the request goes out through the OpenAI adapter. Not a provider value, so
|
|
691
|
+
// it can never reach the database; the dashboard translates it into
|
|
692
|
+
// LLM_PROVIDER_OPENAI_COMPATIBLE on submit.
|
|
693
|
+
const LLM_CATALOG_CUSTOM_KEY = 'custom';
|
|
694
|
+
|
|
695
|
+
// Validation messages for an organization LLM's base URL. Constants rather than
|
|
696
|
+
// inline strings because localizeZodIssue keys its translations on the exact
|
|
697
|
+
// English text, so the two files have to name the same value. Both also carry a
|
|
698
|
+
// word matchStatus recognises ("required", "must be"): the update path re-checks
|
|
699
|
+
// the merged row and throws the message as a plain Error, and anything
|
|
700
|
+
// unrecognised there becomes an opaque 500 instead of a 400.
|
|
701
|
+
const LLM_BASE_URL_REQUIRED_MESSAGE =
|
|
702
|
+
'A base URL is required for an OpenAI-compatible model';
|
|
703
|
+
const LLM_BASE_URL_INVALID_MESSAGE =
|
|
704
|
+
'Base URL must be a public https:// address';
|
|
705
|
+
|
|
660
706
|
const LLM_SYSTEM_DEFAULT = 'SYSTEM_DEFAULT';
|
|
661
707
|
|
|
662
708
|
const MAX_TOOL_LOOPS = 8;
|
|
@@ -831,6 +877,25 @@ const CHUNK_TARGET_CHARS = 2000;
|
|
|
831
877
|
const CHUNK_OVERLAP_CHARS = 200;
|
|
832
878
|
const EMBED_BATCH_SIZE = 96;
|
|
833
879
|
|
|
880
|
+
// Hybrid resource search. Each ranker contributes its top candidates, and the
|
|
881
|
+
// two lists are merged with reciprocal rank fusion: score = sum of 1/(k + rank).
|
|
882
|
+
// Fusion reads ranks, never raw scores, so a cosine distance and a ts_rank never
|
|
883
|
+
// have to be put on one scale. k = 60 is the constant from the original RRF
|
|
884
|
+
// paper; it keeps one list's #1 from outweighing a chunk both lists agree on.
|
|
885
|
+
const RESOURCE_SEARCH_CANDIDATES = 50;
|
|
886
|
+
const RESOURCE_SEARCH_RRF_K = 60;
|
|
887
|
+
// A query word found in more than 1 of every 10 chunks of the artifact is
|
|
888
|
+
// dropped from the lexical query — never below 5 chunks, so a small artifact
|
|
889
|
+
// keeps its words.
|
|
890
|
+
const RESOURCE_SEARCH_COMMON_TERM_DIVISOR = 10;
|
|
891
|
+
const RESOURCE_SEARCH_COMMON_TERM_MIN = 5;
|
|
892
|
+
// Second stage, on wherever the Worker has an AI binding: the top fused
|
|
893
|
+
// candidates are re-scored by a cross-encoder, which reads query and chunk
|
|
894
|
+
// together instead of comparing two independently computed vectors. One model
|
|
895
|
+
// call per search; a failure falls back to the fused order.
|
|
896
|
+
const RESOURCE_SEARCH_RERANK_CANDIDATES = 30;
|
|
897
|
+
const RESOURCE_SEARCH_RERANK_MODEL = '@cf/baai/bge-reranker-base';
|
|
898
|
+
|
|
834
899
|
const RESOURCE_HANDLER_SLEEP_AFTER = '10m';
|
|
835
900
|
|
|
836
901
|
const DOCS_URL = 'https://docs.ganju.ai';
|
|
@@ -1188,8 +1253,7 @@ const MCP_PROXY_TOOL_NAME_MAX = 64;
|
|
|
1188
1253
|
//
|
|
1189
1254
|
// The row's config holds only `activeVersionId` plus the egress/limit settings;
|
|
1190
1255
|
// names and schemas live on the version. That's what keeps MCP boot off the
|
|
1191
|
-
// dispatcher — a slow or broken script must never break tools/list.
|
|
1192
|
-
// docs/CUSTOM_TOOLS.md ("Boot contract").
|
|
1256
|
+
// dispatcher — a slow or broken script must never break tools/list.
|
|
1193
1257
|
const TOOL_DEFINITION_KEY_CUSTOM_CODE = 'custom-code';
|
|
1194
1258
|
|
|
1195
1259
|
// A version is uploaded as `draft`, becomes `published` when it is made active,
|
|
@@ -1977,6 +2041,12 @@ interface PlanLimits {
|
|
|
1977
2041
|
// against). Not used for blocking.
|
|
1978
2042
|
includedMessages: number;
|
|
1979
2043
|
includedEmbeddedBytes: number;
|
|
2044
|
+
// The intervals the hourly job will sync a source on. Empty = no automatic
|
|
2045
|
+
// sync: the org can still sync a source by hand. Refreshing costs crawl or
|
|
2046
|
+
// Drive fetches plus embeddings for whatever changed, all on our side.
|
|
2047
|
+
autoSyncIntervals: ReadonlyArray<
|
|
2048
|
+
typeof RESOURCE_SYNC_INTERVAL_DAILY | typeof RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
2049
|
+
>;
|
|
1980
2050
|
}
|
|
1981
2051
|
|
|
1982
2052
|
const PLAN_LIMITS: Record<
|
|
@@ -2016,7 +2086,8 @@ const PLAN_LIMITS: Record<
|
|
|
2016
2086
|
includedToolCalls: 0,
|
|
2017
2087
|
toolCallHardCap: 10_000,
|
|
2018
2088
|
includedMessages: 100,
|
|
2019
|
-
includedEmbeddedBytes: 5 * MB
|
|
2089
|
+
includedEmbeddedBytes: 5 * MB,
|
|
2090
|
+
autoSyncIntervals: []
|
|
2020
2091
|
},
|
|
2021
2092
|
PRO: {
|
|
2022
2093
|
maxProjects: null,
|
|
@@ -2039,7 +2110,11 @@ const PLAN_LIMITS: Record<
|
|
|
2039
2110
|
includedToolCalls: PRICING_INCLUDED_TOOL_CALLS,
|
|
2040
2111
|
toolCallHardCap: PRICING_TOOL_CALL_HARD_CAP,
|
|
2041
2112
|
includedMessages: PRICING_INCLUDED_MESSAGES,
|
|
2042
|
-
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
|
|
2113
|
+
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
|
|
2114
|
+
autoSyncIntervals: [
|
|
2115
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
2116
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
2117
|
+
]
|
|
2043
2118
|
},
|
|
2044
2119
|
ENTERPRISE: {
|
|
2045
2120
|
maxProjects: null,
|
|
@@ -2062,7 +2137,11 @@ const PLAN_LIMITS: Record<
|
|
|
2062
2137
|
// contract; don't make one plan's abuse limit another plan's product limit.
|
|
2063
2138
|
toolCallHardCap: null,
|
|
2064
2139
|
includedMessages: PRICING_INCLUDED_MESSAGES,
|
|
2065
|
-
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB
|
|
2140
|
+
includedEmbeddedBytes: PRICING_INCLUDED_EMBEDDED_GB * GB,
|
|
2141
|
+
autoSyncIntervals: [
|
|
2142
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
2143
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY
|
|
2144
|
+
]
|
|
2066
2145
|
}
|
|
2067
2146
|
};
|
|
2068
2147
|
|
|
@@ -2081,6 +2160,7 @@ const PLAN_FEATURE_RAW_STORAGE = 'rawStorage' as 'rawStorage';
|
|
|
2081
2160
|
const PLAN_FEATURE_EMBEDDED_STORAGE = 'embeddedStorage' as 'embeddedStorage';
|
|
2082
2161
|
const PLAN_FEATURE_MESSAGE = 'message' as 'message';
|
|
2083
2162
|
const PLAN_FEATURE_TOOL_CALL = 'toolCall' as 'toolCall';
|
|
2163
|
+
const PLAN_FEATURE_RESOURCE_SYNC = 'resourceSync' as 'resourceSync';
|
|
2084
2164
|
|
|
2085
2165
|
// Stable code returned on a quota block so clients can branch on it (402).
|
|
2086
2166
|
const PLAN_LIMIT_ERROR_CODE = 'PLAN_LIMIT_EXCEEDED';
|
|
@@ -2128,7 +2208,7 @@ const POLAR_SERVER_PRODUCTION = 'production' as 'production';
|
|
|
2128
2208
|
const CONSENT_DOCUMENT_TERMS = 'terms' as 'terms';
|
|
2129
2209
|
const CONSENT_DOCUMENT_PRIVACY = 'privacy' as 'privacy';
|
|
2130
2210
|
const CONSENT_DOCUMENTS = [CONSENT_DOCUMENT_TERMS, CONSENT_DOCUMENT_PRIVACY];
|
|
2131
|
-
const CONSENT_CURRENT_VERSION = '2026-
|
|
2211
|
+
const CONSENT_CURRENT_VERSION = '2026-09-10';
|
|
2132
2212
|
|
|
2133
2213
|
const CONSENT_SOURCE_SIGNUP = 'signup' as 'signup';
|
|
2134
2214
|
const CONSENT_SOURCE_REACCEPT = 'reaccept' as 'reaccept';
|
|
@@ -2263,6 +2343,7 @@ export const constants = {
|
|
|
2263
2343
|
PLAN_FEATURE_EMBEDDED_STORAGE,
|
|
2264
2344
|
PLAN_FEATURE_MESSAGE,
|
|
2265
2345
|
PLAN_FEATURE_TOOL_CALL,
|
|
2346
|
+
PLAN_FEATURE_RESOURCE_SYNC,
|
|
2266
2347
|
PLAN_LIMIT_ERROR_CODE,
|
|
2267
2348
|
BILLING_METER_MESSAGES,
|
|
2268
2349
|
BILLING_METER_SHARED_MESSAGES,
|
|
@@ -2457,6 +2538,14 @@ export const constants = {
|
|
|
2457
2538
|
RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER,
|
|
2458
2539
|
RESOURCE_SOURCE_TYPE_CUSTOM_CODE,
|
|
2459
2540
|
RESOURCE_SOURCE_TYPES,
|
|
2541
|
+
RESOURCE_SYNC_INTERVAL_OFF,
|
|
2542
|
+
RESOURCE_SYNC_INTERVAL_DAILY,
|
|
2543
|
+
RESOURCE_SYNC_INTERVAL_WEEKLY,
|
|
2544
|
+
RESOURCE_SYNC_INTERVALS,
|
|
2545
|
+
RESOURCE_SYNC_INTERVAL_DEFAULT,
|
|
2546
|
+
RESOURCE_SYNC_INTERVAL_MS,
|
|
2547
|
+
RESOURCE_SYNC_BATCH_SIZE,
|
|
2548
|
+
RESOURCE_SYNC_MANUAL_COOLDOWN_MS,
|
|
2460
2549
|
CRAWL_RENDERER_CHEERIO,
|
|
2461
2550
|
CRAWL_RENDERER_PLAYWRIGHT,
|
|
2462
2551
|
CRAWL_RENDERERS,
|
|
@@ -2506,6 +2595,9 @@ export const constants = {
|
|
|
2506
2595
|
DEFAULT_LLM_MODEL,
|
|
2507
2596
|
DEFAULT_LLM_SYSTEM_PROMPT,
|
|
2508
2597
|
LLM_CATALOG,
|
|
2598
|
+
LLM_CATALOG_CUSTOM_KEY,
|
|
2599
|
+
LLM_BASE_URL_REQUIRED_MESSAGE,
|
|
2600
|
+
LLM_BASE_URL_INVALID_MESSAGE,
|
|
2509
2601
|
LLM_SYSTEM_DEFAULT,
|
|
2510
2602
|
MAX_TOOL_LOOPS,
|
|
2511
2603
|
CHANNEL_HISTORY_LIMIT,
|
|
@@ -2575,6 +2667,12 @@ export const constants = {
|
|
|
2575
2667
|
CHUNK_TARGET_CHARS,
|
|
2576
2668
|
CHUNK_OVERLAP_CHARS,
|
|
2577
2669
|
EMBED_BATCH_SIZE,
|
|
2670
|
+
RESOURCE_SEARCH_CANDIDATES,
|
|
2671
|
+
RESOURCE_SEARCH_RRF_K,
|
|
2672
|
+
RESOURCE_SEARCH_COMMON_TERM_DIVISOR,
|
|
2673
|
+
RESOURCE_SEARCH_COMMON_TERM_MIN,
|
|
2674
|
+
RESOURCE_SEARCH_RERANK_CANDIDATES,
|
|
2675
|
+
RESOURCE_SEARCH_RERANK_MODEL,
|
|
2578
2676
|
RESOURCE_HANDLER_SLEEP_AFTER,
|
|
2579
2677
|
DOCS_URL,
|
|
2580
2678
|
BASE64_DATA_URI_RE,
|
package/src/index.ts
CHANGED
|
@@ -117,6 +117,11 @@ import type { OAuthProviderConfig } from './oauthProviders';
|
|
|
117
117
|
import { resolveAttachment } from './attachment';
|
|
118
118
|
import { isExposedResource } from './exposedResource';
|
|
119
119
|
import type { ExposableResource } from './exposedResource';
|
|
120
|
+
import { isSyncableResource, resourceSyncProvider } from './syncableResource';
|
|
121
|
+
import type {
|
|
122
|
+
SyncableResource,
|
|
123
|
+
ResourceSyncProvider
|
|
124
|
+
} from './syncableResource';
|
|
120
125
|
import { describeVendorError } from './vendorError';
|
|
121
126
|
import { withDeadline, isDeadlineError } from './deadline';
|
|
122
127
|
import {
|
|
@@ -134,6 +139,11 @@ import type {
|
|
|
134
139
|
ResolveAttachmentResult
|
|
135
140
|
} from './attachment';
|
|
136
141
|
import { ipv4InPrivateRange, isBlockedHost } from './ssrf';
|
|
142
|
+
import {
|
|
143
|
+
requiresLlmBaseUrl,
|
|
144
|
+
isAllowedLlmBaseUrl,
|
|
145
|
+
llmBaseUrlIssue
|
|
146
|
+
} from './llmBaseUrl';
|
|
137
147
|
import { formatRelative } from './formatRelative';
|
|
138
148
|
import { formatRelativeTime } from './formatRelativeTime';
|
|
139
149
|
import { languageCookieDomain } from './languageCookieDomain';
|
|
@@ -181,6 +191,8 @@ import {
|
|
|
181
191
|
import type { Separator, ChunkMetadata, PreparedChunk } from './chunking';
|
|
182
192
|
import { isEmbeddableMimeType } from './embeddable';
|
|
183
193
|
import { l2Normalize } from './l2Normalize';
|
|
194
|
+
import { createResourceReranker } from './resourceRerank';
|
|
195
|
+
import type { RerankAiBinding } from './resourceRerank';
|
|
184
196
|
import { sleep, isRateLimitError, withRateLimitRetry } from './retry';
|
|
185
197
|
import type { RateLimitRetryOptions } from './retry';
|
|
186
198
|
import { processQueueBatch } from './processQueueBatch';
|
|
@@ -199,10 +211,12 @@ import type { ChannelNotifier, ToolStatusEvent } from './channelNotifier';
|
|
|
199
211
|
import {
|
|
200
212
|
resolveDebounceMs,
|
|
201
213
|
channelBufferKey,
|
|
202
|
-
joinBufferedMessages
|
|
214
|
+
joinBufferedMessages,
|
|
215
|
+
batchCommand
|
|
203
216
|
} from './channelDebounce';
|
|
204
217
|
import type {
|
|
205
218
|
BufferedChannelMessage,
|
|
219
|
+
BufferedChannelCommand,
|
|
206
220
|
ChannelBufferEnvelope,
|
|
207
221
|
ChannelBufferFlush
|
|
208
222
|
} from './channelDebounce';
|
|
@@ -286,6 +300,8 @@ export const utils = {
|
|
|
286
300
|
oauthProviders,
|
|
287
301
|
resolveAttachment,
|
|
288
302
|
isExposedResource,
|
|
303
|
+
isSyncableResource,
|
|
304
|
+
resourceSyncProvider,
|
|
289
305
|
describeVendorError,
|
|
290
306
|
withDeadline,
|
|
291
307
|
isDeadlineError,
|
|
@@ -297,6 +313,9 @@ export const utils = {
|
|
|
297
313
|
projectPathIssue,
|
|
298
314
|
ipv4InPrivateRange,
|
|
299
315
|
isBlockedHost,
|
|
316
|
+
requiresLlmBaseUrl,
|
|
317
|
+
isAllowedLlmBaseUrl,
|
|
318
|
+
llmBaseUrlIssue,
|
|
300
319
|
validateMessageVariables,
|
|
301
320
|
formatRelative,
|
|
302
321
|
formatRelativeTime,
|
|
@@ -329,6 +348,7 @@ export const utils = {
|
|
|
329
348
|
prepareChunks,
|
|
330
349
|
isEmbeddableMimeType,
|
|
331
350
|
l2Normalize,
|
|
351
|
+
createResourceReranker,
|
|
332
352
|
sleep,
|
|
333
353
|
isRateLimitError,
|
|
334
354
|
withRateLimitRetry,
|
|
@@ -336,6 +356,7 @@ export const utils = {
|
|
|
336
356
|
getToolStatusMessage,
|
|
337
357
|
resolveDebounceMs,
|
|
338
358
|
channelBufferKey,
|
|
359
|
+
batchCommand,
|
|
339
360
|
joinBufferedMessages,
|
|
340
361
|
isResourceSourceEnabled,
|
|
341
362
|
isDownloadableSource,
|
|
@@ -399,12 +420,14 @@ export type {
|
|
|
399
420
|
ExtractedDocumentMetadata,
|
|
400
421
|
ExtractedDocumentSource,
|
|
401
422
|
RateLimitRetryOptions,
|
|
423
|
+
RerankAiBinding,
|
|
402
424
|
QueueBatchLike,
|
|
403
425
|
QueueMessageLike,
|
|
404
426
|
ProcessQueueBatchHandlers,
|
|
405
427
|
ChannelNotifier,
|
|
406
428
|
ToolStatusEvent,
|
|
407
429
|
BufferedChannelMessage,
|
|
430
|
+
BufferedChannelCommand,
|
|
408
431
|
ChannelBufferEnvelope,
|
|
409
432
|
ChannelBufferFlush,
|
|
410
433
|
Source,
|
|
@@ -430,6 +453,8 @@ export type {
|
|
|
430
453
|
ResolvedAttachment,
|
|
431
454
|
ResolveAttachmentResult,
|
|
432
455
|
ExposableResource,
|
|
456
|
+
SyncableResource,
|
|
457
|
+
ResourceSyncProvider,
|
|
433
458
|
PlanLimitDetails,
|
|
434
459
|
PlanLimits
|
|
435
460
|
};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { constants } from './constants';
|
|
2
|
+
import { isBlockedHost } from './ssrf';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The rules an `organization_llm.base_url` has to satisfy, in one place.
|
|
6
|
+
*
|
|
7
|
+
* Two write paths need them and neither can see the whole picture alone: the
|
|
8
|
+
* create schema validates a complete payload, while an update is a partial, so
|
|
9
|
+
* the provider may arrive in the request and the base URL may only exist on the
|
|
10
|
+
* stored row. Keeping the rules here means the two can't drift into disagreeing
|
|
11
|
+
* about what a valid row is — the same reason every other write in this package
|
|
12
|
+
* goes through a shared validator rather than an inline check.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Which providers need a base URL of their own.
|
|
17
|
+
*
|
|
18
|
+
* Every other provider has an endpoint baked into its SDK. `openai-compatible`
|
|
19
|
+
* has none: it is the escape hatch for services that merely speak the OpenAI
|
|
20
|
+
* protocol, so without a base URL the adapter would send the customer's key to
|
|
21
|
+
* `api.openai.com` and fail somewhere far from the form they typed it into.
|
|
22
|
+
*/
|
|
23
|
+
export const requiresLlmBaseUrl = (provider?: string | null): boolean =>
|
|
24
|
+
provider === constants.LLM_PROVIDER_OPENAI_COMPATIBLE;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Whether a base URL is safe to send a request to.
|
|
28
|
+
*
|
|
29
|
+
* This is an outbound fetch target chosen by a customer, on a worker holding
|
|
30
|
+
* their decrypted API key, so it gets the same screening as every other
|
|
31
|
+
* caller-supplied URL: `https` only, because the key travels in a header, and
|
|
32
|
+
* no loopback, private or link-local host. The Workers runtime can't resolve
|
|
33
|
+
* DNS, so — as everywhere else `isBlockedHost` is used — this screens literal
|
|
34
|
+
* hosts and IPs and is not a defense against DNS rebinding.
|
|
35
|
+
*/
|
|
36
|
+
export const isAllowedLlmBaseUrl = (value: string): boolean => {
|
|
37
|
+
let parsed: URL;
|
|
38
|
+
try {
|
|
39
|
+
parsed = new URL(value);
|
|
40
|
+
} catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
if (parsed.protocol !== 'https:') return false;
|
|
44
|
+
return !isBlockedHost(parsed.hostname);
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Both rules against one provider/base-URL pair: the message to reject with, or
|
|
49
|
+
* `null` when the pair is acceptable. The messages are constants so
|
|
50
|
+
* `localizeZodIssue` can translate them, and both carry a word `matchStatus`
|
|
51
|
+
* recognises — the update path throws this as a plain Error rather than a
|
|
52
|
+
* ZodError, and an unrecognised message there answers 500 where it means 400.
|
|
53
|
+
*/
|
|
54
|
+
export const llmBaseUrlIssue = (
|
|
55
|
+
provider?: string | null,
|
|
56
|
+
baseUrl?: string | null
|
|
57
|
+
): string | null => {
|
|
58
|
+
if (requiresLlmBaseUrl(provider) && !baseUrl) {
|
|
59
|
+
return constants.LLM_BASE_URL_REQUIRED_MESSAGE;
|
|
60
|
+
}
|
|
61
|
+
if (baseUrl && !isAllowedLlmBaseUrl(baseUrl)) {
|
|
62
|
+
return constants.LLM_BASE_URL_INVALID_MESSAGE;
|
|
63
|
+
}
|
|
64
|
+
return null;
|
|
65
|
+
};
|
package/src/localizeZodIssue.ts
CHANGED
|
@@ -74,6 +74,12 @@ const CUSTOM_MESSAGES: Record<string, Record<string, string>> = {
|
|
|
74
74
|
},
|
|
75
75
|
[constants.CUSTOM_CODE_RESOURCE_FILE_TOO_LARGE_MESSAGE]: {
|
|
76
76
|
es: `Los bytes del archivo superan el límite de ${constants.CUSTOM_CODE_MAX_RESOURCE_FILE_BYTES / (1024 * 1024)}MB`
|
|
77
|
+
},
|
|
78
|
+
[constants.LLM_BASE_URL_REQUIRED_MESSAGE]: {
|
|
79
|
+
es: 'Se requiere una URL base para un modelo compatible con OpenAI'
|
|
80
|
+
},
|
|
81
|
+
[constants.LLM_BASE_URL_INVALID_MESSAGE]: {
|
|
82
|
+
es: 'La URL base debe ser una dirección https:// pública'
|
|
77
83
|
}
|
|
78
84
|
};
|
|
79
85
|
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { constants } from './constants';
|
|
2
|
+
import type { EnvSource } from './getEnv';
|
|
3
|
+
|
|
4
|
+
// The slice of the Workers AI binding the reranker calls. Declared here rather
|
|
5
|
+
// than imported so this package doesn't take a dependency on workers-types.
|
|
6
|
+
export interface RerankAiBinding {
|
|
7
|
+
run(
|
|
8
|
+
model: string,
|
|
9
|
+
input: { query: string; contexts: Array<{ text: string }>; top_k?: number }
|
|
10
|
+
): Promise<unknown>;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
type RerankResponse = {
|
|
14
|
+
response?: Array<{ id?: number; score?: number }>;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The cross-encoder second stage for resource search, or undefined when the
|
|
19
|
+
* Worker has no `AI` binding — local dev without Workers AI keeps plain fusion.
|
|
20
|
+
*
|
|
21
|
+
* Returns one score per document, in input order. A response that doesn't
|
|
22
|
+
* cover every document returns null, which keeps the fused order — a partial
|
|
23
|
+
* ranking would silently drop the chunks it left out.
|
|
24
|
+
*/
|
|
25
|
+
export const createResourceReranker = (
|
|
26
|
+
source: EnvSource
|
|
27
|
+
):
|
|
28
|
+
| ((query: string, documents: string[]) => Promise<number[] | null>)
|
|
29
|
+
| undefined => {
|
|
30
|
+
const ai = (source.env as { AI?: RerankAiBinding } | undefined)?.AI;
|
|
31
|
+
if (!ai) return undefined;
|
|
32
|
+
|
|
33
|
+
return async (query, documents) => {
|
|
34
|
+
const result = (await ai.run(constants.RESOURCE_SEARCH_RERANK_MODEL, {
|
|
35
|
+
query,
|
|
36
|
+
contexts: documents.map(text => ({ text })),
|
|
37
|
+
top_k: documents.length
|
|
38
|
+
})) as RerankResponse;
|
|
39
|
+
|
|
40
|
+
const scores: Array<number | undefined> = new Array(documents.length);
|
|
41
|
+
for (const item of result?.response ?? []) {
|
|
42
|
+
if (
|
|
43
|
+
typeof item.id === 'number' &&
|
|
44
|
+
typeof item.score === 'number' &&
|
|
45
|
+
item.id >= 0 &&
|
|
46
|
+
item.id < documents.length
|
|
47
|
+
) {
|
|
48
|
+
scores[item.id] = item.score;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (scores.some(score => score === undefined)) return null;
|
|
52
|
+
return scores as number[];
|
|
53
|
+
};
|
|
54
|
+
};
|
package/src/schema.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
|
|
3
3
|
import { constants } from './constants';
|
|
4
|
+
import { isAllowedLlmBaseUrl, requiresLlmBaseUrl } from './llmBaseUrl';
|
|
4
5
|
import { isReservedSlug, isValidSlugFormat } from './slug';
|
|
5
6
|
import { isReservedToolName } from './reservedToolName';
|
|
6
7
|
import { slugifyTitle } from './slugifyTitle';
|
|
@@ -572,11 +573,33 @@ const ORGANIZATION_LIST_LLM = z.object({
|
|
|
572
573
|
organizationId: z.uuid()
|
|
573
574
|
});
|
|
574
575
|
|
|
575
|
-
|
|
576
|
+
// The two rules live in `llmBaseUrl.ts`, because the update controller applies
|
|
577
|
+
// them again to the merged row — the only place both halves are visible. Here
|
|
578
|
+
// they are just the field validator and the cross-field check.
|
|
579
|
+
const LLM_BASE_URL = z
|
|
580
|
+
.url()
|
|
581
|
+
.max(500)
|
|
582
|
+
.refine(isAllowedLlmBaseUrl, constants.LLM_BASE_URL_INVALID_MESSAGE);
|
|
583
|
+
|
|
584
|
+
const BASE_URL_REQUIRED = {
|
|
585
|
+
message: constants.LLM_BASE_URL_REQUIRED_MESSAGE,
|
|
586
|
+
path: ['baseUrl']
|
|
587
|
+
};
|
|
588
|
+
|
|
589
|
+
const hasBaseUrlWhenRequired = (
|
|
590
|
+
provider: string | undefined,
|
|
591
|
+
baseUrl: unknown
|
|
592
|
+
) => !requiresLlmBaseUrl(provider) || !!baseUrl;
|
|
593
|
+
|
|
594
|
+
// The base-URL rule is a cross-field check, and zod cannot `.omit()` from a
|
|
595
|
+
// schema that already carries one. So the shape stays a plain object — the
|
|
596
|
+
// `_VIEW` variants below derive from it — and the refinement is applied last,
|
|
597
|
+
// to each schema in turn, which also keeps the two in step.
|
|
598
|
+
const ORGANIZATION_CREATE_LLM_SHAPE = z.object({
|
|
576
599
|
name: z.string().min(1).max(200),
|
|
577
600
|
provider: z.enum(constants.LLM_PROVIDERS),
|
|
578
601
|
model: z.string().min(1).max(200),
|
|
579
|
-
baseUrl:
|
|
602
|
+
baseUrl: LLM_BASE_URL.optional().or(z.literal('')),
|
|
580
603
|
apiKey: z.string().min(1).max(500),
|
|
581
604
|
systemPrompt: z.string().max(10000).optional(),
|
|
582
605
|
config: z.record(z.string(), z.any()).optional(),
|
|
@@ -584,12 +607,21 @@ const ORGANIZATION_CREATE_LLM = z.object({
|
|
|
584
607
|
organizationId: z.uuid()
|
|
585
608
|
});
|
|
586
609
|
|
|
587
|
-
const
|
|
610
|
+
const ORGANIZATION_CREATE_LLM = ORGANIZATION_CREATE_LLM_SHAPE.refine(
|
|
611
|
+
v => hasBaseUrlWhenRequired(v.provider, v.baseUrl),
|
|
612
|
+
BASE_URL_REQUIRED
|
|
613
|
+
);
|
|
614
|
+
|
|
615
|
+
// An update is a partial: an omitted field keeps its stored value, so the
|
|
616
|
+
// base-URL rule can only be checked here when the caller sends the provider and
|
|
617
|
+
// the base URL together. The controller re-checks it against the merged row,
|
|
618
|
+
// which is the only place both halves are known.
|
|
619
|
+
const ORGANIZATION_UPDATE_LLM_SHAPE = z.object({
|
|
588
620
|
llmId: z.uuid(),
|
|
589
621
|
name: z.string().min(1).max(200).optional(),
|
|
590
622
|
provider: z.enum(constants.LLM_PROVIDERS).optional(),
|
|
591
623
|
model: z.string().min(1).max(200).optional(),
|
|
592
|
-
baseUrl:
|
|
624
|
+
baseUrl: LLM_BASE_URL.optional().or(z.literal('')).nullable(),
|
|
593
625
|
apiKey: z.string().min(1).max(500).optional(),
|
|
594
626
|
systemPrompt: z.string().max(10000).optional().nullable(),
|
|
595
627
|
config: z.record(z.string(), z.any()).optional().nullable(),
|
|
@@ -597,6 +629,21 @@ const ORGANIZATION_UPDATE_LLM = z.object({
|
|
|
597
629
|
organizationId: z.uuid()
|
|
598
630
|
});
|
|
599
631
|
|
|
632
|
+
const refineUpdateBaseUrl = <
|
|
633
|
+
T extends z.ZodType<{ provider?: string; baseUrl?: unknown }>
|
|
634
|
+
>(
|
|
635
|
+
schema: T
|
|
636
|
+
) =>
|
|
637
|
+
schema.refine(
|
|
638
|
+
v =>
|
|
639
|
+
v.baseUrl === undefined || hasBaseUrlWhenRequired(v.provider, v.baseUrl),
|
|
640
|
+
BASE_URL_REQUIRED
|
|
641
|
+
);
|
|
642
|
+
|
|
643
|
+
const ORGANIZATION_UPDATE_LLM = refineUpdateBaseUrl(
|
|
644
|
+
ORGANIZATION_UPDATE_LLM_SHAPE
|
|
645
|
+
);
|
|
646
|
+
|
|
600
647
|
const ORGANIZATION_REMOVE_LLM = z.object({
|
|
601
648
|
llmId: z.uuid(),
|
|
602
649
|
userId: z.uuid(),
|
|
@@ -649,16 +696,21 @@ const ACCESS_TOKEN_CREATE_VIEW = ACCESS_TOKEN_CREATE.omit({
|
|
|
649
696
|
projectId: true
|
|
650
697
|
});
|
|
651
698
|
|
|
652
|
-
const ORGANIZATION_CREATE_LLM_VIEW =
|
|
699
|
+
const ORGANIZATION_CREATE_LLM_VIEW = ORGANIZATION_CREATE_LLM_SHAPE.omit({
|
|
653
700
|
userId: true,
|
|
654
701
|
organizationId: true
|
|
655
|
-
})
|
|
702
|
+
}).refine(
|
|
703
|
+
v => hasBaseUrlWhenRequired(v.provider, v.baseUrl),
|
|
704
|
+
BASE_URL_REQUIRED
|
|
705
|
+
);
|
|
656
706
|
|
|
657
|
-
const ORGANIZATION_UPDATE_LLM_VIEW =
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
707
|
+
const ORGANIZATION_UPDATE_LLM_VIEW = refineUpdateBaseUrl(
|
|
708
|
+
ORGANIZATION_UPDATE_LLM_SHAPE.omit({
|
|
709
|
+
llmId: true,
|
|
710
|
+
userId: true,
|
|
711
|
+
organizationId: true
|
|
712
|
+
})
|
|
713
|
+
);
|
|
662
714
|
|
|
663
715
|
// `channel.config` is an open bag, so this stays loose — it only pins down the
|
|
664
716
|
// keys we actually read, and passes everything else through untouched.
|
|
@@ -1397,6 +1449,21 @@ const ARTIFACT_UPDATE_RESOURCE_SHOW_SOURCE = z.object({
|
|
|
1397
1449
|
organizationId: z.uuid()
|
|
1398
1450
|
});
|
|
1399
1451
|
|
|
1452
|
+
const ARTIFACT_UPDATE_RESOURCE_SYNC_INTERVAL = z.object({
|
|
1453
|
+
resourceId: z.uuid(),
|
|
1454
|
+
syncInterval: z.enum(constants.RESOURCE_SYNC_INTERVALS),
|
|
1455
|
+
projectId: z.uuid(),
|
|
1456
|
+
userId: z.uuid(),
|
|
1457
|
+
organizationId: z.uuid()
|
|
1458
|
+
});
|
|
1459
|
+
|
|
1460
|
+
const ARTIFACT_SYNC_RESOURCE = z.object({
|
|
1461
|
+
resourceId: z.uuid(),
|
|
1462
|
+
projectId: z.uuid(),
|
|
1463
|
+
userId: z.uuid(),
|
|
1464
|
+
organizationId: z.uuid()
|
|
1465
|
+
});
|
|
1466
|
+
|
|
1400
1467
|
const ARTIFACT_GET = z.object({
|
|
1401
1468
|
projectId: z.uuid(),
|
|
1402
1469
|
userId: z.uuid(),
|
|
@@ -1573,6 +1640,8 @@ export const Schema = {
|
|
|
1573
1640
|
CHANNEL_LIST_CONVERSATIONS,
|
|
1574
1641
|
CHANNEL_LIST_MESSAGES,
|
|
1575
1642
|
ARTIFACT_UPDATE_RESOURCE_SHOW_SOURCE,
|
|
1643
|
+
ARTIFACT_UPDATE_RESOURCE_SYNC_INTERVAL,
|
|
1644
|
+
ARTIFACT_SYNC_RESOURCE,
|
|
1576
1645
|
HTTP_ENDPOINT_CONFIG,
|
|
1577
1646
|
HTTP_ENDPOINT_CONFIG_WRITE,
|
|
1578
1647
|
MCP_PROXY_CONFIG,
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { constants } from './constants';
|
|
2
|
+
|
|
3
|
+
// Which resources are sync roots: the rows a sync starts from and records its
|
|
4
|
+
// schedule on.
|
|
5
|
+
//
|
|
6
|
+
// - a WEBSITE seed (the row with no parent) — a sync re-crawls the site;
|
|
7
|
+
// - a Drive/OneDrive folder — a sync walks it for new, changed and removed
|
|
8
|
+
// files;
|
|
9
|
+
// - a Drive/OneDrive file imported on its own — a sync re-fetches it when it
|
|
10
|
+
// changed.
|
|
11
|
+
//
|
|
12
|
+
// A page under a website or a file under a folder is not a root: its parent's
|
|
13
|
+
// sync covers it. An uploaded file has nothing to sync from.
|
|
14
|
+
export interface SyncableResource {
|
|
15
|
+
sourceType?: string | null;
|
|
16
|
+
parentResourceId?: string | null;
|
|
17
|
+
metadata?: unknown;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export type ResourceSyncProvider = 'website' | 'google-drive' | 'one-drive';
|
|
21
|
+
|
|
22
|
+
export const resourceSyncProvider = (
|
|
23
|
+
resource: SyncableResource
|
|
24
|
+
): ResourceSyncProvider | null => {
|
|
25
|
+
if (resource.parentResourceId) return null;
|
|
26
|
+
switch (resource.sourceType) {
|
|
27
|
+
case constants.RESOURCE_SOURCE_TYPE_WEBSITE:
|
|
28
|
+
return 'website';
|
|
29
|
+
case constants.RESOURCE_SOURCE_TYPE_GOOGLE_DRIVE_FOLDER:
|
|
30
|
+
return 'google-drive';
|
|
31
|
+
case constants.RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER:
|
|
32
|
+
return 'one-drive';
|
|
33
|
+
case constants.RESOURCE_SOURCE_TYPE_FILE: {
|
|
34
|
+
const meta =
|
|
35
|
+
resource.metadata && typeof resource.metadata === 'object'
|
|
36
|
+
? (resource.metadata as Record<string, unknown>)
|
|
37
|
+
: null;
|
|
38
|
+
if (typeof meta?.driveFileId === 'string') return 'google-drive';
|
|
39
|
+
if (typeof meta?.oneDriveItemId === 'string') return 'one-drive';
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
default:
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export const isSyncableResource = (resource: SyncableResource): boolean =>
|
|
48
|
+
resourceSyncProvider(resource) !== null;
|
package/src/toolCatalog.ts
CHANGED
|
@@ -469,7 +469,7 @@ export const TOOL_CATALOG = [
|
|
|
469
469
|
key: 'search-resources',
|
|
470
470
|
title: 'Search Resources',
|
|
471
471
|
description:
|
|
472
|
-
'Find the resources most relevant to a question
|
|
472
|
+
'Find the resources most relevant to a question, by meaning and by exact keywords.',
|
|
473
473
|
requiredScopes: null
|
|
474
474
|
},
|
|
475
475
|
{
|