@exulu/backend 1.66.0 → 1.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-VPSLTGZF.js +10267 -0
- package/dist/{convert-exulu-tools-to-ai-sdk-tools-ZFIN7A5V.js → convert-exulu-tools-to-ai-sdk-tools-CHQF36XW.js} +1 -2
- package/dist/index.cjs +23930 -22308
- package/dist/index.d.cts +401 -100
- package/dist/index.d.ts +401 -100
- package/dist/index.js +15215 -4233
- package/ee/agentic-retrieval/v3/agent-loop.ts +4 -4
- package/ee/agentic-retrieval/v3/index.ts +20 -6
- package/ee/python/documents/processing/doc_processor.ts +79 -34
- package/ee/python/requirements.txt +8 -1
- package/ee/python/setup.sh +0 -49
- package/ee/queues/decorator.ts +36 -0
- package/ee/queues/prune-job-results.ts +55 -0
- package/ee/schemas.ts +19 -0
- package/ee/workers.ts +59 -32
- package/package.json +1 -1
- package/dist/chunk-KQDNL5WU.js +0 -19399
- package/ee/agentic-retrieval/v4/agent-loop.ts +0 -208
- package/ee/agentic-retrieval/v4/context-sampler.ts +0 -79
- package/ee/agentic-retrieval/v4/index.ts +0 -690
- package/ee/agentic-retrieval/v4/types.ts +0 -58
- package/ee/python/.hermes/.env.example +0 -8
- package/ee/python/.hermes/README.md +0 -44
- package/ee/python/.hermes/SOUL.md.example +0 -8
- package/ee/python/.hermes/config.yaml.example +0 -55
package/dist/index.d.ts
CHANGED
|
@@ -10,6 +10,29 @@ import { z } from 'zod';
|
|
|
10
10
|
import { Tiktoken } from 'tiktoken/lite';
|
|
11
11
|
import models from 'tiktoken/model_to_encoding.json';
|
|
12
12
|
|
|
13
|
+
interface Project {
|
|
14
|
+
id: string;
|
|
15
|
+
name: string;
|
|
16
|
+
description: string;
|
|
17
|
+
custom_instructions: string;
|
|
18
|
+
rights_mode?: 'private' | 'users' | 'roles' | 'public';
|
|
19
|
+
created_by?: string;
|
|
20
|
+
project_items?: string[];
|
|
21
|
+
RBAC?: {
|
|
22
|
+
type?: string;
|
|
23
|
+
users?: Array<{
|
|
24
|
+
id: string;
|
|
25
|
+
rights: 'read' | 'write';
|
|
26
|
+
}>;
|
|
27
|
+
roles?: Array<{
|
|
28
|
+
id: string;
|
|
29
|
+
rights: 'read' | 'write';
|
|
30
|
+
}>;
|
|
31
|
+
};
|
|
32
|
+
createdAt?: string;
|
|
33
|
+
updatedAt?: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
13
36
|
type ApiKeyScopeMode = "admin" | "agents";
|
|
14
37
|
type User = {
|
|
15
38
|
id: number;
|
|
@@ -22,10 +45,20 @@ type User = {
|
|
|
22
45
|
personal_system_prompt?: string;
|
|
23
46
|
super_admin?: boolean;
|
|
24
47
|
favourite_agents?: string[];
|
|
48
|
+
/** Per-user favourited data items — global ids ("<contextId>/<itemId>"). */
|
|
49
|
+
favourite_items?: string[];
|
|
50
|
+
/** Per-user recently viewed data items — global ids, most-recent first. */
|
|
51
|
+
recently_viewed_items?: string[];
|
|
25
52
|
scope_mode?: ApiKeyScopeMode;
|
|
26
53
|
agent_ids?: string[];
|
|
27
54
|
role: UserRole;
|
|
28
55
|
team?: ExuluTeam;
|
|
56
|
+
/**
|
|
57
|
+
* Optional attribution target (mainly for API keys, type "api"): hydrated
|
|
58
|
+
* from the `project` uuid column at auth time so buildTags can emit
|
|
59
|
+
* project_id_ for API-triggered requests.
|
|
60
|
+
*/
|
|
61
|
+
project?: Project;
|
|
29
62
|
/**
|
|
30
63
|
* Live LiteLLM budget snapshot for the user, attached at context time when
|
|
31
64
|
* the "show user budget in chat" setting is on. Not a Postgres column.
|
|
@@ -184,7 +217,47 @@ interface Item {
|
|
|
184
217
|
[key: string]: any;
|
|
185
218
|
}
|
|
186
219
|
|
|
187
|
-
|
|
220
|
+
/**
|
|
221
|
+
* OAuth 2.0 configuration for an {@link ExuluTool}. When a tool is constructed
|
|
222
|
+
* with an `oauth` property, Exulu wraps its `execute` so it only runs when a
|
|
223
|
+
* valid access token exists for the calling (toolId, userId) pair. When no
|
|
224
|
+
* valid token exists the tool short-circuits and returns an authorization URL
|
|
225
|
+
* the agent can show the user; the generic /oauth/callback route completes the
|
|
226
|
+
* flow and persists the tokens.
|
|
227
|
+
*
|
|
228
|
+
* Only the standard authorization-code grant is supported. All values are
|
|
229
|
+
* declared in code (source them from env vars or however you like) — none of
|
|
230
|
+
* them are exposed as admin-configurable tool config.
|
|
231
|
+
*/
|
|
232
|
+
type ExuluOauthConfig = {
|
|
233
|
+
/** The provider's authorization endpoint, e.g. https://app.hubspot.com/oauth/authorize */
|
|
234
|
+
authorizationUrl: string;
|
|
235
|
+
/** The provider's token endpoint, e.g. https://api.hubapi.com/oauth/v1/token */
|
|
236
|
+
tokenUrl: string;
|
|
237
|
+
clientId: string;
|
|
238
|
+
/** Never leaves the server: used only in the server-side token exchange. */
|
|
239
|
+
clientSecret: string;
|
|
240
|
+
/** Scopes to request; joined with spaces in the authorization URL. */
|
|
241
|
+
scopes: string[];
|
|
242
|
+
/** PKCE (S256). Defaults to true; set false for providers that reject PKCE. */
|
|
243
|
+
pkce?: boolean;
|
|
244
|
+
/**
|
|
245
|
+
* Extra query params appended to the authorization URL, e.g.
|
|
246
|
+
* `{ access_type: "offline", prompt: "consent" }` to make Google return a
|
|
247
|
+
* refresh token.
|
|
248
|
+
*/
|
|
249
|
+
extraAuthParams?: Record<string, string>;
|
|
250
|
+
};
|
|
251
|
+
/** The oauth context injected into an oauth-enabled tool's execute inputs. */
|
|
252
|
+
type ExuluOauthToolContext = {
|
|
253
|
+
accessToken: string;
|
|
254
|
+
/** null when the provider did not report an expiry. */
|
|
255
|
+
expiresAt: Date | null;
|
|
256
|
+
/** Space-joined scopes the token was granted, when reported by the provider. */
|
|
257
|
+
scopes: string | null;
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
declare const PUBLIC_TOOL_TYPES: readonly ["function", "web_search", "skill", "context"];
|
|
188
261
|
type PublicToolType = (typeof PUBLIC_TOOL_TYPES)[number];
|
|
189
262
|
type ToolType = PublicToolType | "agent" | "context";
|
|
190
263
|
declare class ExuluTool {
|
|
@@ -196,13 +269,14 @@ declare class ExuluTool {
|
|
|
196
269
|
type: ToolType;
|
|
197
270
|
tool: Tool;
|
|
198
271
|
needsApproval: boolean;
|
|
272
|
+
oauth?: ExuluOauthConfig;
|
|
199
273
|
config: {
|
|
200
274
|
name: string;
|
|
201
275
|
description: string;
|
|
202
276
|
type: "boolean" | "string" | "number" | "variable";
|
|
203
277
|
default?: string | boolean | number;
|
|
204
278
|
}[];
|
|
205
|
-
constructor({ id, name, description, category, inputSchema, type, execute, config, needsApproval, }: {
|
|
279
|
+
constructor({ id, name, description, category, inputSchema, type, execute, config, needsApproval, oauth, }: {
|
|
206
280
|
id: string;
|
|
207
281
|
name: string;
|
|
208
282
|
description: string;
|
|
@@ -216,6 +290,7 @@ declare class ExuluTool {
|
|
|
216
290
|
default?: string | boolean | number;
|
|
217
291
|
}[];
|
|
218
292
|
needsApproval?: boolean;
|
|
293
|
+
oauth?: ExuluOauthConfig;
|
|
219
294
|
execute: (inputs: any, options?: any) => Promise<{
|
|
220
295
|
result?: string;
|
|
221
296
|
job?: string;
|
|
@@ -308,6 +383,50 @@ type ExuluContextProcessor = {
|
|
|
308
383
|
};
|
|
309
384
|
};
|
|
310
385
|
|
|
386
|
+
/**
|
|
387
|
+
* Chunking is now an ExuluContext concern (it used to live on the removed
|
|
388
|
+
* ExuluEmbedder class). A context may supply its own `chunker` to control how
|
|
389
|
+
* an item is split into embeddable chunks; if it doesn't, `defaultChunker`
|
|
390
|
+
* runs. Embedding generation itself goes through LiteLLM via resolveEmbedder —
|
|
391
|
+
* the chunker only produces the text segments.
|
|
392
|
+
*/
|
|
393
|
+
type ChunkerResponse = {
|
|
394
|
+
item: Item & {
|
|
395
|
+
id: string;
|
|
396
|
+
};
|
|
397
|
+
chunks: {
|
|
398
|
+
content: string;
|
|
399
|
+
index: number;
|
|
400
|
+
metadata?: Record<string, unknown>;
|
|
401
|
+
}[];
|
|
402
|
+
};
|
|
403
|
+
/**
|
|
404
|
+
* A chunker takes a (fully-hydrated) item and a target max chunk size and
|
|
405
|
+
* returns the ordered text chunks to embed. `utils.storage` is provided for
|
|
406
|
+
* chunkers that need to read file contents from object storage.
|
|
407
|
+
*
|
|
408
|
+
* Note: unlike the old ExuluEmbedder.chunker, there is no `settings` argument —
|
|
409
|
+
* the per-context `embedder_settings` config layer was removed. Chunkers that
|
|
410
|
+
* need configuration should close over it in code.
|
|
411
|
+
*/
|
|
412
|
+
type ChunkerOperation = (item: Item & {
|
|
413
|
+
id: string;
|
|
414
|
+
}, maxChunkSize: number, utils: {
|
|
415
|
+
storage: ExuluStorage;
|
|
416
|
+
}) => Promise<ChunkerResponse>;
|
|
417
|
+
/**
|
|
418
|
+
* Built-in chunker used when a context configures an embedder model but does
|
|
419
|
+
* not provide its own `chunker`. It runs the standard SentenceChunker (also
|
|
420
|
+
* exposed as ExuluChunkers.sentence) over the item's primary text — preferring
|
|
421
|
+
* a `content` field, then `description`, combined with the `name` — so a
|
|
422
|
+
* context "just works" from a model name alone. `maxChunkSize` is used as the
|
|
423
|
+
* per-chunk token budget. Contexts with structured or file-backed content
|
|
424
|
+
* should supply a custom ChunkerOperation.
|
|
425
|
+
*/
|
|
426
|
+
declare const defaultChunker: ChunkerOperation;
|
|
427
|
+
|
|
428
|
+
type ExuluRightsMode = "private" | "users" | "roles" | "teams" | "public";
|
|
429
|
+
|
|
311
430
|
type STATISTICS_TYPE = "CONTEXT_RETRIEVE" | "SOURCE_UPDATE" | "EMBEDDER_UPSERT" | "EMBEDDER_GENERATE" | "EMBEDDER_DELETE" | "WORKFLOW_RUN" | "CONTEXT_UPSERT" | "TOOL_CALL" | "AGENT_RUN";
|
|
312
431
|
declare const STATISTICS_TYPE_ENUM: {
|
|
313
432
|
CONTEXT_RETRIEVE: string;
|
|
@@ -331,67 +450,6 @@ type ExuluStatistic = {
|
|
|
331
450
|
};
|
|
332
451
|
type STATISTICS_LABELS = "tool" | "agent" | "flow" | "api" | "claude-code" | "user" | "processor";
|
|
333
452
|
|
|
334
|
-
type ExuluEmbedderConfig = {
|
|
335
|
-
name: string;
|
|
336
|
-
description: string;
|
|
337
|
-
default?: string;
|
|
338
|
-
};
|
|
339
|
-
type VectorGenerationResponse = Promise<{
|
|
340
|
-
id: string;
|
|
341
|
-
chunks: {
|
|
342
|
-
content: string;
|
|
343
|
-
index: number;
|
|
344
|
-
metadata: Record<string, string>;
|
|
345
|
-
vector: number[];
|
|
346
|
-
}[];
|
|
347
|
-
}>;
|
|
348
|
-
type VectorGenerateOperation = (inputs: ChunkerResponse, settings: Record<string, string>) => VectorGenerationResponse;
|
|
349
|
-
type ChunkerOperation = (item: Item & {
|
|
350
|
-
id: string;
|
|
351
|
-
}, maxChunkSize: number, utils: {
|
|
352
|
-
storage: ExuluStorage;
|
|
353
|
-
}, config: Record<string, string>) => Promise<ChunkerResponse>;
|
|
354
|
-
type ChunkerResponse = {
|
|
355
|
-
item: Item & {
|
|
356
|
-
id: string;
|
|
357
|
-
};
|
|
358
|
-
chunks: {
|
|
359
|
-
content: string;
|
|
360
|
-
index: number;
|
|
361
|
-
}[];
|
|
362
|
-
};
|
|
363
|
-
declare class ExuluEmbedder {
|
|
364
|
-
id: string;
|
|
365
|
-
name: string;
|
|
366
|
-
slug: string;
|
|
367
|
-
queue?: Promise<ExuluQueueConfig>;
|
|
368
|
-
private generateEmbeddings;
|
|
369
|
-
description: string;
|
|
370
|
-
vectorDimensions: number;
|
|
371
|
-
config?: ExuluEmbedderConfig[];
|
|
372
|
-
maxChunkSize: number;
|
|
373
|
-
_chunker: ChunkerOperation;
|
|
374
|
-
constructor({ id, name, description, generateEmbeddings, queue, vectorDimensions, maxChunkSize, chunker, config, }: {
|
|
375
|
-
id: string;
|
|
376
|
-
name: string;
|
|
377
|
-
description: string;
|
|
378
|
-
config?: ExuluEmbedderConfig[];
|
|
379
|
-
generateEmbeddings: VectorGenerateOperation;
|
|
380
|
-
chunker: ChunkerOperation;
|
|
381
|
-
queue?: Promise<ExuluQueueConfig>;
|
|
382
|
-
vectorDimensions: number;
|
|
383
|
-
maxChunkSize: number;
|
|
384
|
-
});
|
|
385
|
-
chunker: (context: string, item: Item & {
|
|
386
|
-
id: string;
|
|
387
|
-
}, maxChunkSize: number, config: ExuluConfig) => Promise<ChunkerResponse>;
|
|
388
|
-
private hydrateEmbedderConfig;
|
|
389
|
-
generateFromQuery(context: string, query: string, statistics?: ExuluStatisticParams, user?: number, role?: string): VectorGenerationResponse;
|
|
390
|
-
generateFromDocument(context: string, input: Item, config: ExuluConfig, statistics?: ExuluStatisticParams, user?: number, role?: string): VectorGenerationResponse;
|
|
391
|
-
}
|
|
392
|
-
|
|
393
|
-
type ExuluRightsMode = "private" | "users" | "roles" | "public";
|
|
394
|
-
|
|
395
453
|
/**
|
|
396
454
|
* Base operator type with comparison operations
|
|
397
455
|
*/
|
|
@@ -475,6 +533,68 @@ declare const VectorMethodEnum: {
|
|
|
475
533
|
};
|
|
476
534
|
type VectorMethod = (typeof VectorMethodEnum)[keyof typeof VectorMethodEnum];
|
|
477
535
|
|
|
536
|
+
/**
|
|
537
|
+
* A single entity type a context extracts (e.g. { name: "Person", description: "..." }).
|
|
538
|
+
* Declared in code on the ExuluContext `entities.types`, and/or in the DB via the
|
|
539
|
+
* `entity_type_settings` admin table. The effective set is the union of both.
|
|
540
|
+
*/
|
|
541
|
+
type EntityTypeDefinition = {
|
|
542
|
+
name: string;
|
|
543
|
+
description: string;
|
|
544
|
+
};
|
|
545
|
+
/**
|
|
546
|
+
* The opt-in entity-layer configuration on an ExuluContext. When this block is
|
|
547
|
+
* absent (and no admin types exist for the context) the entity layer is OFF and
|
|
548
|
+
* retrieval/ingestion behave exactly as before.
|
|
549
|
+
*/
|
|
550
|
+
type ExuluEntitiesConfig = {
|
|
551
|
+
/** Entity types declared in code. Merged (union) with admin-declared types. */
|
|
552
|
+
types?: EntityTypeDefinition[];
|
|
553
|
+
/** models.id used for extraction. Resolved via resolveModel(). Falls back to a platform default. */
|
|
554
|
+
model?: string;
|
|
555
|
+
/** Weight of the shared-entity boost term in retrieval ranking. Default 0.3. */
|
|
556
|
+
boostWeight?: number;
|
|
557
|
+
/** Drop mentions below this extractor confidence (0..1). Default 0.5. */
|
|
558
|
+
confidenceThreshold?: number;
|
|
559
|
+
/** Target language for canonical entity names. Default "english". */
|
|
560
|
+
canonicalLanguage?: string;
|
|
561
|
+
};
|
|
562
|
+
/** A related entity surfaced via derived co-occurrence. */
|
|
563
|
+
type RelatedEntity = {
|
|
564
|
+
id: string;
|
|
565
|
+
name: string;
|
|
566
|
+
type: string;
|
|
567
|
+
/** Relatedness weight (normalized Jaccard over shared documents). */
|
|
568
|
+
weight: number;
|
|
569
|
+
};
|
|
570
|
+
/** Entity intelligence for one query entity, returned to the calling agent. */
|
|
571
|
+
type QueryEntityInsight = {
|
|
572
|
+
id: string;
|
|
573
|
+
type: string;
|
|
574
|
+
name: string;
|
|
575
|
+
/** How many of the returned chunks mention this entity. */
|
|
576
|
+
matchedInResults: number;
|
|
577
|
+
/** Distinct documents in the context mentioning this entity (maintained counter). */
|
|
578
|
+
relatedDocCount: number;
|
|
579
|
+
/** Top-K co-occurring entities, weighted. */
|
|
580
|
+
relatedEntities: RelatedEntity[];
|
|
581
|
+
};
|
|
582
|
+
type EntityInsights = {
|
|
583
|
+
queryEntities: QueryEntityInsight[];
|
|
584
|
+
};
|
|
585
|
+
/** Caller-supplied entity filter for agent-driven exploration. */
|
|
586
|
+
type EntityFilter = {
|
|
587
|
+
/** Resolve directly by entity id. */
|
|
588
|
+
entityIds?: string[];
|
|
589
|
+
/** Or resolve by (type, name) — name is normalized to a canonical key. */
|
|
590
|
+
entities?: {
|
|
591
|
+
type: string;
|
|
592
|
+
name: string;
|
|
593
|
+
}[];
|
|
594
|
+
/** "any" (default): chunk mentions at least one. "all": chunk mentions all. */
|
|
595
|
+
mode?: "any" | "all";
|
|
596
|
+
};
|
|
597
|
+
|
|
478
598
|
type VectorSearchChunkResult = {
|
|
479
599
|
chunk_content: string;
|
|
480
600
|
chunk_index: number;
|
|
@@ -491,12 +611,30 @@ type VectorSearchChunkResult = {
|
|
|
491
611
|
chunk_cosine_distance?: number;
|
|
492
612
|
chunk_fts_rank?: number;
|
|
493
613
|
chunk_hybrid_score?: number;
|
|
614
|
+
/** Entities mentioned in this chunk (present only when the entity layer is on). */
|
|
615
|
+
chunk_entities?: {
|
|
616
|
+
id: string;
|
|
617
|
+
name: string;
|
|
618
|
+
type: string;
|
|
619
|
+
}[];
|
|
494
620
|
context?: {
|
|
495
621
|
name: string;
|
|
496
622
|
id: string;
|
|
497
623
|
};
|
|
498
624
|
};
|
|
499
625
|
|
|
626
|
+
/**
|
|
627
|
+
* A context's embedder is now just a reference to a LiteLLM embedding model
|
|
628
|
+
* (plus an optional queue), not an ExuluEmbedder instance. Embedding generation
|
|
629
|
+
* goes through resolveEmbedder; chunking is configured separately via the
|
|
630
|
+
* context's `chunker` (or the built-in default chunker).
|
|
631
|
+
*/
|
|
632
|
+
type ExuluContextEmbedder = {
|
|
633
|
+
/** LiteLLM model_name of the embedding model (declared in config.litellm.yaml). */
|
|
634
|
+
model: string;
|
|
635
|
+
/** When set, embedding generation runs as a background job on this queue. */
|
|
636
|
+
queue?: Promise<ExuluQueueConfig>;
|
|
637
|
+
};
|
|
500
638
|
type ExuluContextFieldDefinition = {
|
|
501
639
|
name: string;
|
|
502
640
|
type: ExuluFieldTypes;
|
|
@@ -539,14 +677,20 @@ declare class ExuluContext {
|
|
|
539
677
|
fields: ExuluContextFieldDefinition[];
|
|
540
678
|
processor?: ExuluContextProcessor;
|
|
541
679
|
description: string;
|
|
542
|
-
embedder?:
|
|
680
|
+
embedder?: ExuluContextEmbedder;
|
|
681
|
+
/**
|
|
682
|
+
* Splits an item into embeddable chunks. Moved here from the removed
|
|
683
|
+
* ExuluEmbedder. When omitted, the built-in `defaultChunker` (SentenceChunker)
|
|
684
|
+
* is used so a context works from just an embedder model name.
|
|
685
|
+
*/
|
|
686
|
+
chunker?: ChunkerOperation;
|
|
543
687
|
queryRewriter?: (query: string) => Promise<string>;
|
|
544
688
|
resultReranker?: (results: {
|
|
545
689
|
chunk_content: string;
|
|
546
690
|
chunk_index: number;
|
|
547
691
|
chunk_id: string;
|
|
548
692
|
chunk_source: string;
|
|
549
|
-
chunk_metadata: Record<string,
|
|
693
|
+
chunk_metadata: Record<string, unknown>;
|
|
550
694
|
chunk_created_at: string;
|
|
551
695
|
chunk_updated_at: string;
|
|
552
696
|
item_id: string;
|
|
@@ -557,7 +701,7 @@ declare class ExuluContext {
|
|
|
557
701
|
chunk_index: number;
|
|
558
702
|
chunk_id: string;
|
|
559
703
|
chunk_source: string;
|
|
560
|
-
chunk_metadata: Record<string,
|
|
704
|
+
chunk_metadata: Record<string, unknown>;
|
|
561
705
|
chunk_created_at: string;
|
|
562
706
|
chunk_updated_at: string;
|
|
563
707
|
item_id: string;
|
|
@@ -579,13 +723,20 @@ declare class ExuluContext {
|
|
|
579
723
|
};
|
|
580
724
|
languages?: ("german" | "english")[];
|
|
581
725
|
};
|
|
726
|
+
/**
|
|
727
|
+
* Optional entity-layer configuration. When present (or when an admin has
|
|
728
|
+
* configured entity types for this context) the graph/entity retrieval
|
|
729
|
+
* features are switched on. Absent → identical behavior to before.
|
|
730
|
+
*/
|
|
731
|
+
entities?: ExuluEntitiesConfig;
|
|
582
732
|
sources: ExuluContextSource[];
|
|
583
|
-
constructor({ id, name, description, embedder, processor, active, fields, queryRewriter, resultReranker, configuration, sources, }: {
|
|
733
|
+
constructor({ id, name, description, embedder, chunker, processor, active, fields, queryRewriter, resultReranker, configuration, entities, sources, }: {
|
|
584
734
|
id: string;
|
|
585
735
|
name: string;
|
|
586
736
|
fields: ExuluContextFieldDefinition[];
|
|
587
737
|
description: string;
|
|
588
|
-
embedder?:
|
|
738
|
+
embedder?: ExuluContextEmbedder;
|
|
739
|
+
chunker?: ChunkerOperation;
|
|
589
740
|
sources: ExuluContextSource[];
|
|
590
741
|
category?: string;
|
|
591
742
|
active: boolean;
|
|
@@ -607,6 +758,7 @@ declare class ExuluContext {
|
|
|
607
758
|
hybrid?: number;
|
|
608
759
|
};
|
|
609
760
|
};
|
|
761
|
+
entities?: ExuluEntitiesConfig;
|
|
610
762
|
});
|
|
611
763
|
processField: (trigger: STATISTICS_LABELS, item: Item, exuluConfig: ExuluConfig, user?: number, role?: string) => Promise<{
|
|
612
764
|
result: Item | undefined;
|
|
@@ -633,6 +785,7 @@ declare class ExuluContext {
|
|
|
633
785
|
before?: number;
|
|
634
786
|
after?: number;
|
|
635
787
|
};
|
|
788
|
+
entityFilter?: EntityFilter;
|
|
636
789
|
}) => Promise<{
|
|
637
790
|
itemFilters: SearchFilters;
|
|
638
791
|
chunkFilters: SearchFilters;
|
|
@@ -645,6 +798,7 @@ declare class ExuluContext {
|
|
|
645
798
|
embedder: string;
|
|
646
799
|
};
|
|
647
800
|
chunks: VectorSearchChunkResult[];
|
|
801
|
+
entityInsights?: EntityInsights;
|
|
648
802
|
}>;
|
|
649
803
|
deleteAll: () => Promise<{
|
|
650
804
|
count: number;
|
|
@@ -697,30 +851,46 @@ declare class ExuluContext {
|
|
|
697
851
|
}>;
|
|
698
852
|
};
|
|
699
853
|
};
|
|
854
|
+
/**
|
|
855
|
+
* Entity-layer administration: backfill extraction over existing items,
|
|
856
|
+
* count stale items (for the admin "run backfill?" prompt), and purge a type.
|
|
857
|
+
*/
|
|
858
|
+
entityLayer: {
|
|
859
|
+
/** Count items whose entities were extracted with an out-of-date type set. */
|
|
860
|
+
countStale: () => Promise<number>;
|
|
861
|
+
/**
|
|
862
|
+
* Full re-extraction over items. `onlyStale` (default true) limits to items
|
|
863
|
+
* whose type-set signature is out of date. Runs inline in batches with a
|
|
864
|
+
* safeguard cap; entity extraction (not re-embedding) is what runs here.
|
|
865
|
+
*/
|
|
866
|
+
backfill: ({ onlyStale, limit, }?: {
|
|
867
|
+
onlyStale?: boolean;
|
|
868
|
+
limit?: number;
|
|
869
|
+
}) => Promise<{
|
|
870
|
+
processed: number;
|
|
871
|
+
skipped: number;
|
|
872
|
+
}>;
|
|
873
|
+
/**
|
|
874
|
+
* Extract + ingest entities for a SINGLE item — powers the item detail
|
|
875
|
+
* page's "Extract entities" test action. Returns the number of mentions
|
|
876
|
+
* found so the UI can report the result.
|
|
877
|
+
*/
|
|
878
|
+
extractItem: (itemId: string) => Promise<{
|
|
879
|
+
extracted: number;
|
|
880
|
+
}>;
|
|
881
|
+
/** Detach all entities from a single item (drops links, prunes orphans). */
|
|
882
|
+
detachItem: (itemId: string) => Promise<{
|
|
883
|
+
detached: number;
|
|
884
|
+
}>;
|
|
885
|
+
/** Remove all entities (and their mentions via cascade) of a given type. */
|
|
886
|
+
purgeType: (typeName: string) => Promise<{
|
|
887
|
+
removed: number;
|
|
888
|
+
}>;
|
|
889
|
+
};
|
|
700
890
|
createItemsTable: () => Promise<void>;
|
|
701
891
|
createChunksTable: () => Promise<void>;
|
|
702
892
|
}
|
|
703
893
|
|
|
704
|
-
declare class ExuluReranker {
|
|
705
|
-
id: string;
|
|
706
|
-
name: string;
|
|
707
|
-
description: string;
|
|
708
|
-
execute: (params: {
|
|
709
|
-
query: string;
|
|
710
|
-
chunks: VectorSearchChunkResult[];
|
|
711
|
-
}) => Promise<VectorSearchChunkResult[]>;
|
|
712
|
-
constructor({ id, name, description, execute, }: {
|
|
713
|
-
id: string;
|
|
714
|
-
name: string;
|
|
715
|
-
description: string;
|
|
716
|
-
execute: (params: {
|
|
717
|
-
query: string;
|
|
718
|
-
chunks: VectorSearchChunkResult[];
|
|
719
|
-
}) => Promise<VectorSearchChunkResult[]>;
|
|
720
|
-
});
|
|
721
|
-
run(query: string, chunks: VectorSearchChunkResult[]): Promise<VectorSearchChunkResult[]>;
|
|
722
|
-
}
|
|
723
|
-
|
|
724
894
|
type ExuluAgentToolConfig = {
|
|
725
895
|
id: string;
|
|
726
896
|
type: string;
|
|
@@ -799,8 +969,8 @@ declare class ExuluProvider {
|
|
|
799
969
|
constructor({ id, name, description, config, capabilities, type, maxContextLength, provider, queue, authenticationInformation, workflows, }: ExuluProviderParams);
|
|
800
970
|
get providerName(): string;
|
|
801
971
|
get modelName(): string;
|
|
802
|
-
tool: (instance: string, providers: ExuluProvider[], contexts: ExuluContext[]
|
|
803
|
-
generateSync: ({ prompt, req, user, session, inputMessages, approvedTools, currentTools, currentSkills, allExuluTools, statistics, toolConfigs, providerapikey, languageModel, contexts,
|
|
972
|
+
tool: (instance: string, providers: ExuluProvider[], contexts: ExuluContext[]) => Promise<ExuluTool | null>;
|
|
973
|
+
generateSync: ({ prompt, req, user, session, inputMessages, approvedTools, currentTools, currentSkills, allExuluTools, statistics, toolConfigs, providerapikey, languageModel, contexts, exuluConfig, agent, instructions, maxStepCount, onTokenUsage }: {
|
|
804
974
|
prompt?: string;
|
|
805
975
|
user?: User;
|
|
806
976
|
maxStepCount?: number;
|
|
@@ -817,7 +987,6 @@ declare class ExuluProvider {
|
|
|
817
987
|
providerapikey?: string | undefined;
|
|
818
988
|
languageModel: LanguageModel;
|
|
819
989
|
contexts?: ExuluContext[] | undefined;
|
|
820
|
-
rerankers?: ExuluReranker[] | undefined;
|
|
821
990
|
exuluConfig?: ExuluConfig;
|
|
822
991
|
instructions?: string;
|
|
823
992
|
onTokenUsage?: (usage: {
|
|
@@ -833,7 +1002,7 @@ declare class ExuluProvider {
|
|
|
833
1002
|
* - Image files -> image parts (which ARE supported by Responses API)
|
|
834
1003
|
*/
|
|
835
1004
|
private processFilePartsInMessages;
|
|
836
|
-
generateStream: ({ user, session, agent, message, previousMessages, currentTools, currentSkills, approvedTools, allExuluTools, toolConfigs, providerapikey, languageModel, contexts,
|
|
1005
|
+
generateStream: ({ user, session, agent, message, previousMessages, currentTools, currentSkills, approvedTools, allExuluTools, toolConfigs, providerapikey, languageModel, contexts, exuluConfig, instructions, req, maxStepCount }: {
|
|
837
1006
|
user?: User;
|
|
838
1007
|
session?: string;
|
|
839
1008
|
agent?: ExuluAgent;
|
|
@@ -848,7 +1017,6 @@ declare class ExuluProvider {
|
|
|
848
1017
|
providerapikey?: string | undefined;
|
|
849
1018
|
languageModel: LanguageModel;
|
|
850
1019
|
contexts?: ExuluContext[] | undefined;
|
|
851
|
-
rerankers?: ExuluReranker[] | undefined;
|
|
852
1020
|
exuluConfig?: ExuluConfig;
|
|
853
1021
|
instructions?: string;
|
|
854
1022
|
req?: Request;
|
|
@@ -954,17 +1122,15 @@ declare class ExuluApp {
|
|
|
954
1122
|
private _config?;
|
|
955
1123
|
private _evals;
|
|
956
1124
|
private _queues;
|
|
957
|
-
private _rerankers;
|
|
958
1125
|
private _contexts?;
|
|
959
1126
|
private _tools;
|
|
960
1127
|
private _expressApp;
|
|
961
1128
|
constructor();
|
|
962
|
-
create: ({ contexts, providers, config, agents, tools, evals,
|
|
1129
|
+
create: ({ contexts, providers, config, agents, tools, evals, }: {
|
|
963
1130
|
contexts?: Record<string, ExuluContext>;
|
|
964
1131
|
config: ExuluConfig;
|
|
965
1132
|
agents?: ExuluAgent[];
|
|
966
1133
|
providers?: ExuluProvider[];
|
|
967
|
-
rerankers?: ExuluReranker[];
|
|
968
1134
|
evals?: ExuluEval[];
|
|
969
1135
|
tools?: ExuluTool[];
|
|
970
1136
|
}) => Promise<ExuluApp>;
|
|
@@ -2063,14 +2229,72 @@ declare function validatePythonEnvironment(packageRoot?: string, checkPackages?:
|
|
|
2063
2229
|
message: string;
|
|
2064
2230
|
}>;
|
|
2065
2231
|
|
|
2232
|
+
/**
|
|
2233
|
+
* resolveOcr — the OCR-side counterpart of resolveEmbedder.
|
|
2234
|
+
*
|
|
2235
|
+
* Like resolveEmbedder, this is LiteLLM-ONLY: OCR always goes through the
|
|
2236
|
+
* spawned LiteLLM proxy's Mistral-compatible `/v1/ocr` endpoint. There is no
|
|
2237
|
+
* in-code provider/SDK fallback — a caller just names a LiteLLM `model` (e.g.
|
|
2238
|
+
* "mistral-ocr", "vertex-ocr") and it works, with cost attribution via tags
|
|
2239
|
+
* (user/role/project/agent/routine/context, when provided — see buildTags()).
|
|
2240
|
+
*
|
|
2241
|
+
* Routing OCR through the proxy means we can cost-control it through the same
|
|
2242
|
+
* tag-based budgets as chat and embeddings, and switch the underlying provider
|
|
2243
|
+
* (mistral / azure_ai / vertex_ai) by editing config.litellm.yaml without
|
|
2244
|
+
* touching this code.
|
|
2245
|
+
*
|
|
2246
|
+
* LiteLLM follows the Mistral OCR request/response shape:
|
|
2247
|
+
* https://docs.mistral.ai/capabilities/vision/#optical-character-recognition-ocr
|
|
2248
|
+
*/
|
|
2249
|
+
type ResolveOcrInput = {
|
|
2250
|
+
/** LiteLLM model_name of the OCR model (e.g. "mistral-ocr"). */
|
|
2251
|
+
model: string;
|
|
2252
|
+
/** Context this OCR belongs to — emitted as context_id_/context_name_ tags. */
|
|
2253
|
+
contextId?: string;
|
|
2254
|
+
contextName?: string;
|
|
2255
|
+
user?: User;
|
|
2256
|
+
/** When only a numeric user id is available (background ingestion jobs). */
|
|
2257
|
+
userId?: number;
|
|
2258
|
+
roleId?: string;
|
|
2259
|
+
project?: Project;
|
|
2260
|
+
agent?: ExuluAgent;
|
|
2261
|
+
routine?: {
|
|
2262
|
+
id: string;
|
|
2263
|
+
name: string;
|
|
2264
|
+
};
|
|
2265
|
+
};
|
|
2266
|
+
|
|
2066
2267
|
type DocumentProcessorConfig = {
|
|
2067
2268
|
vlm?: {
|
|
2068
|
-
|
|
2269
|
+
/**
|
|
2270
|
+
* LiteLLM model_name for the VLM page-validation pass (declared in
|
|
2271
|
+
* config.litellm.yaml, e.g. "vertex-gemini-2.5-flash"). Resolved via
|
|
2272
|
+
* resolveModel() so the VLM pass shares the same tag-based cost controls
|
|
2273
|
+
* and provider-switching as chat / embeddings / OCR, and the underlying
|
|
2274
|
+
* provider can be swapped without code changes.
|
|
2275
|
+
*/
|
|
2276
|
+
model: string;
|
|
2069
2277
|
concurrency: number;
|
|
2070
2278
|
};
|
|
2071
2279
|
processor: {
|
|
2072
2280
|
name: "docling" | "liteparse" | "mistral" | "officeparser";
|
|
2281
|
+
/**
|
|
2282
|
+
* LiteLLM model_name for the "mistral" OCR processor (declared in
|
|
2283
|
+
* config.litellm.yaml). Defaults to "mistral-ocr". OCR is routed through
|
|
2284
|
+
* the LiteLLM proxy so it shares the same tag-based cost controls as chat
|
|
2285
|
+
* and embeddings, and the underlying provider (mistral / azure_ai /
|
|
2286
|
+
* vertex_ai) can be switched without code changes.
|
|
2287
|
+
*/
|
|
2288
|
+
model?: string;
|
|
2073
2289
|
};
|
|
2290
|
+
/**
|
|
2291
|
+
* Optional cost-attribution context, forwarded to LiteLLM as spend tags
|
|
2292
|
+
* (user / role / project / context) for both the OCR pass (resolveOcr) and
|
|
2293
|
+
* the VLM page-validation pass (resolveModel). Not yet populated by callers;
|
|
2294
|
+
* the wiring is in place so per-user/per-context budgets work the moment
|
|
2295
|
+
* attribution is threaded through.
|
|
2296
|
+
*/
|
|
2297
|
+
attribution?: Omit<ResolveOcrInput, "model">;
|
|
2074
2298
|
debugging?: {
|
|
2075
2299
|
deleteTempFiles?: boolean;
|
|
2076
2300
|
};
|
|
@@ -2090,6 +2314,81 @@ declare function documentProcessor({ file, name, config }: {
|
|
|
2090
2314
|
config?: DocumentProcessorConfig;
|
|
2091
2315
|
}): Promise<ProcessedDocument | undefined>;
|
|
2092
2316
|
|
|
2317
|
+
/**
|
|
2318
|
+
* resolveReranker — the rerank-side counterpart of resolveEmbedder / resolveOcr.
|
|
2319
|
+
*
|
|
2320
|
+
* Like those, this is LiteLLM-ONLY: reranking always goes through the spawned
|
|
2321
|
+
* LiteLLM proxy's cohere-compatible `/v1/rerank` endpoint. There is no in-code
|
|
2322
|
+
* provider/SDK fallback — a caller just names a LiteLLM `model` (a model_name
|
|
2323
|
+
* declared in config.litellm.yaml with `model_info.type: reranker`) and it
|
|
2324
|
+
* works, with cost attribution via tags (user/role/project/agent/routine/
|
|
2325
|
+
* context, when provided — see buildTags()).
|
|
2326
|
+
*
|
|
2327
|
+
* Routing rerank through the proxy means we cost-control it through the same
|
|
2328
|
+
* tag-based budgets as chat / embeddings / OCR, and switch the underlying
|
|
2329
|
+
* provider (cohere / vertex_ai / together_ai / ...) by editing
|
|
2330
|
+
* config.litellm.yaml without touching this code.
|
|
2331
|
+
*
|
|
2332
|
+
* `rerank` takes the chunks directly, builds each document as
|
|
2333
|
+
* `item_name + ": " + chunk_content` (the standard retrieval convention), calls
|
|
2334
|
+
* the proxy, and maps the relevance scores back onto the chunks (reordered,
|
|
2335
|
+
* `rerank_score` attached). The item type is constrained structurally
|
|
2336
|
+
* (item_name / chunk_content) so both ChunkResult and VectorSearchChunkResult
|
|
2337
|
+
* work without importing either — no src/exulu → retrieval/GraphQL dependency.
|
|
2338
|
+
*
|
|
2339
|
+
* LiteLLM follows the Cohere rerank request/response shape:
|
|
2340
|
+
* https://docs.cohere.com/reference/rerank
|
|
2341
|
+
*/
|
|
2342
|
+
type ResolveRerankerInput = {
|
|
2343
|
+
/** LiteLLM model_name of the reranker (e.g. "rerank-v4.0-pro"). */
|
|
2344
|
+
model: string;
|
|
2345
|
+
/** Context this rerank belongs to — emitted as context_id_/context_name_ tags. */
|
|
2346
|
+
contextId?: string;
|
|
2347
|
+
contextName?: string;
|
|
2348
|
+
user?: User;
|
|
2349
|
+
/** When only a numeric user id is available (background ingestion jobs). */
|
|
2350
|
+
userId?: number;
|
|
2351
|
+
roleId?: string;
|
|
2352
|
+
project?: Project;
|
|
2353
|
+
agent?: ExuluAgent;
|
|
2354
|
+
routine?: {
|
|
2355
|
+
id: string;
|
|
2356
|
+
name: string;
|
|
2357
|
+
};
|
|
2358
|
+
};
|
|
2359
|
+
/** Minimal chunk shape rerank() needs to build a document. */
|
|
2360
|
+
type RerankableChunk = {
|
|
2361
|
+
item_name?: string;
|
|
2362
|
+
chunk_content?: string;
|
|
2363
|
+
};
|
|
2364
|
+
|
|
2365
|
+
/**
|
|
2366
|
+
* Public, package-facing reranker — the counterpart of
|
|
2367
|
+
* `ExuluDocumentProcessor.process`. A drop-in replacement for a hand-rolled
|
|
2368
|
+
* Cohere / Google reranker: pass `{ query, items, model }` and get the items
|
|
2369
|
+
* back reordered desc by relevance with a `rerank_score` attached.
|
|
2370
|
+
*
|
|
2371
|
+
* `model` is a LiteLLM model_name declared in config.litellm.yaml with
|
|
2372
|
+
* `model_info.type: reranker`, so the SAME call works against any supported
|
|
2373
|
+
* provider (cohere / vertex_ai / together_ai / ...) — switch providers in
|
|
2374
|
+
* config, not in code — and reranking is cost-attributed via the optional
|
|
2375
|
+
* identity/context fields (user / role / project / agent / routine / context).
|
|
2376
|
+
*
|
|
2377
|
+
* Each document is built as `item_name + ": " + chunk_content` (the standard
|
|
2378
|
+
* retrieval convention). Items are constrained structurally, so any chunk shape
|
|
2379
|
+
* carrying `item_name` / `chunk_content` works and the extra fields are
|
|
2380
|
+
* preserved on the returned objects.
|
|
2381
|
+
*/
|
|
2382
|
+
type ExuluRerankInput<T extends RerankableChunk> = {
|
|
2383
|
+
query: string;
|
|
2384
|
+
items: T[];
|
|
2385
|
+
/** Only score/return the top N items (optional optimization hint). */
|
|
2386
|
+
topN?: number;
|
|
2387
|
+
} & ResolveRerankerInput;
|
|
2388
|
+
declare function rerank<T extends RerankableChunk>(input: ExuluRerankInput<T>): Promise<(T & {
|
|
2389
|
+
rerank_score: number;
|
|
2390
|
+
})[]>;
|
|
2391
|
+
|
|
2093
2392
|
/**
|
|
2094
2393
|
* Creates the v3 ExuluTool for agentic context retrieval.
|
|
2095
2394
|
*
|
|
@@ -2099,9 +2398,8 @@ declare function documentProcessor({ file, name, config }: {
|
|
|
2099
2398
|
* - Context example records sampled at init and cached
|
|
2100
2399
|
* - Strategy-specific instructions and tool sets
|
|
2101
2400
|
*/
|
|
2102
|
-
declare function createAgenticRetrievalToolV3({ contexts, instructions: adminInstructions,
|
|
2401
|
+
declare function createAgenticRetrievalToolV3({ contexts, instructions: adminInstructions, user, role, model, preselected, memoryItems }: {
|
|
2103
2402
|
contexts: ExuluContext[];
|
|
2104
|
-
rerankers: ExuluReranker[];
|
|
2105
2403
|
user?: User;
|
|
2106
2404
|
role?: string;
|
|
2107
2405
|
model?: LanguageModel;
|
|
@@ -2181,6 +2479,9 @@ declare const ExuluAuthentication: {
|
|
|
2181
2479
|
declare const ExuluDocumentProcessor: {
|
|
2182
2480
|
process: typeof documentProcessor;
|
|
2183
2481
|
};
|
|
2482
|
+
declare const ExuluReranker: {
|
|
2483
|
+
rerank: typeof rerank;
|
|
2484
|
+
};
|
|
2184
2485
|
declare const ExuluOtel: {
|
|
2185
2486
|
create: ({ SIGNOZ_ACCESS_TOKEN, SIGNOZ_TRACES_URL, SIGNOZ_LOGS_URL, }: {
|
|
2186
2487
|
SIGNOZ_ACCESS_TOKEN: string;
|
|
@@ -2221,4 +2522,4 @@ declare const ExuluPython: {
|
|
|
2221
2522
|
instructions: typeof getPythonSetupInstructions;
|
|
2222
2523
|
};
|
|
2223
2524
|
|
|
2224
|
-
export { type JOB_STATUS as EXULU_JOB_STATUS, JOB_STATUS_ENUM as EXULU_JOB_STATUS_ENUM, type STATISTICS_TYPE as EXULU_STATISTICS_TYPE, STATISTICS_TYPE_ENUM as EXULU_STATISTICS_TYPE_ENUM, type ExuluAgent, ExuluApp, ExuluAuthentication, ExuluChunkers, ExuluContext, ExuluDatabase, ExuluDefaultProviders, ExuluDefaultTools, ExuluDocumentProcessor,
|
|
2525
|
+
export { type ChunkerOperation, type ChunkerResponse, type JOB_STATUS as EXULU_JOB_STATUS, JOB_STATUS_ENUM as EXULU_JOB_STATUS_ENUM, type STATISTICS_TYPE as EXULU_STATISTICS_TYPE, STATISTICS_TYPE_ENUM as EXULU_STATISTICS_TYPE_ENUM, type ExuluAgent, ExuluApp, ExuluAuthentication, ExuluChunkers, ExuluContext, type ExuluContextEmbedder, ExuluDatabase, ExuluDefaultProviders, ExuluDefaultTools, ExuluDocumentProcessor, ExuluEval, type Item as ExuluItem, ExuluJobs, type ExuluOauthConfig, type ExuluOauthToolContext, ExuluOtel, ExuluProvider, ExuluPython, queues as ExuluQueues, ExuluReranker, ExuluTool, trajectoryRegistry as ExuluTrajectoryRegistry, ExuluVariables, defaultChunker };
|