@exulu/backend 1.68.0 → 1.69.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-VPSLTGZF.js → chunk-PJSDLFXL.js} +167 -48
- package/dist/{convert-exulu-tools-to-ai-sdk-tools-CHQF36XW.js → convert-exulu-tools-to-ai-sdk-tools-MXLGIOCT.js} +1 -1
- package/dist/index.cjs +592 -113
- package/dist/index.d.cts +60 -2
- package/dist/index.d.ts +60 -2
- package/dist/index.js +394 -52
- package/ee/python/documents/processing/doc_processor.ts +61 -10
- package/ee/python/documents/processing/split_pdf.py +97 -0
- package/ee/queues/queues.ts +10 -0
- package/ee/queues/redis-startup.ts +121 -0
- package/ee/workers.ts +6 -0
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as _opentelemetry_sdk_node from '@opentelemetry/sdk-node';
|
|
2
2
|
import * as knex from 'knex';
|
|
3
|
+
import { Knex } from 'knex';
|
|
3
4
|
import { RedisClientType } from 'redis';
|
|
4
5
|
import * as bullmq from 'bullmq';
|
|
5
6
|
import { Queue } from 'bullmq';
|
|
@@ -148,6 +149,7 @@ interface ExuluAgent {
|
|
|
148
149
|
instructions?: string;
|
|
149
150
|
feedback?: boolean;
|
|
150
151
|
suggestions_enabled?: boolean;
|
|
152
|
+
sandbox_enabled?: boolean;
|
|
151
153
|
slug?: string;
|
|
152
154
|
tools?: {
|
|
153
155
|
id: string;
|
|
@@ -786,6 +788,9 @@ declare class ExuluContext {
|
|
|
786
788
|
after?: number;
|
|
787
789
|
};
|
|
788
790
|
entityFilter?: EntityFilter;
|
|
791
|
+
/** Precomputed query embedding; forwarded to vectorSearch to skip re-embedding (caller must use
|
|
792
|
+
* this context's embedding model). Spread into vectorSearch via `...options` below. */
|
|
793
|
+
queryEmbedding?: number[];
|
|
789
794
|
}) => Promise<{
|
|
790
795
|
itemFilters: SearchFilters;
|
|
791
796
|
chunkFilters: SearchFilters;
|
|
@@ -828,9 +833,11 @@ declare class ExuluContext {
|
|
|
828
833
|
getItem: ({ item }: {
|
|
829
834
|
item: Item;
|
|
830
835
|
}) => Promise<Item>;
|
|
831
|
-
getItems: ({ filters, fields, }: {
|
|
836
|
+
getItems: ({ filters, fields, user, role, }: {
|
|
832
837
|
filters?: any[];
|
|
833
838
|
fields?: string[];
|
|
839
|
+
user?: User;
|
|
840
|
+
role?: string;
|
|
834
841
|
}) => Promise<Item[]>;
|
|
835
842
|
embeddings: {
|
|
836
843
|
generate: {
|
|
@@ -1804,6 +1811,50 @@ declare class RecursiveChunker extends BaseChunker {
|
|
|
1804
1811
|
toString(): string;
|
|
1805
1812
|
}
|
|
1806
1813
|
|
|
1814
|
+
interface AuthorizedReadOpts {
|
|
1815
|
+
itemIds?: string[];
|
|
1816
|
+
externalIds?: string[];
|
|
1817
|
+
chunkIndexRange?: {
|
|
1818
|
+
from?: number;
|
|
1819
|
+
to?: number;
|
|
1820
|
+
};
|
|
1821
|
+
}
|
|
1822
|
+
/**
|
|
1823
|
+
* Supported, RBAC-safe read API for retrieval clients (e.g. the agentic harness).
|
|
1824
|
+
* Relevance + visibility still go through ExuluContext.search(); this namespace
|
|
1825
|
+
* adds the read shapes search() cannot express, always over authorized rows.
|
|
1826
|
+
*/
|
|
1827
|
+
declare const ExuluReadApi: {
|
|
1828
|
+
getTableName: (id: string) => string;
|
|
1829
|
+
getChunksTableName: (id: string) => string;
|
|
1830
|
+
getEntitiesTableName: (id: string) => string;
|
|
1831
|
+
getChunkEntitiesTableName: (id: string) => string;
|
|
1832
|
+
entitiesAvailable: (context: {
|
|
1833
|
+
id: string;
|
|
1834
|
+
entities?: unknown;
|
|
1835
|
+
}) => Promise<boolean>;
|
|
1836
|
+
authorizedRead: (context: {
|
|
1837
|
+
id: string;
|
|
1838
|
+
fields?: unknown[];
|
|
1839
|
+
entities?: unknown;
|
|
1840
|
+
}, user: User, role: string, opts?: AuthorizedReadOpts) => Promise<VectorSearchChunkResult[]>;
|
|
1841
|
+
embedQuery: (context: {
|
|
1842
|
+
id: string;
|
|
1843
|
+
name?: string;
|
|
1844
|
+
embedder: {
|
|
1845
|
+
model: string;
|
|
1846
|
+
};
|
|
1847
|
+
}, text: string, opts?: {
|
|
1848
|
+
user?: User;
|
|
1849
|
+
role?: string;
|
|
1850
|
+
inputType?: "document" | "query";
|
|
1851
|
+
}) => Promise<number[]>;
|
|
1852
|
+
};
|
|
1853
|
+
|
|
1854
|
+
declare function postgresClient(): Promise<{
|
|
1855
|
+
db: Knex;
|
|
1856
|
+
}>;
|
|
1857
|
+
|
|
1807
1858
|
/**
|
|
1808
1859
|
* Represents the essential data for a sentence within a text.
|
|
1809
1860
|
*
|
|
@@ -2286,6 +2337,13 @@ type DocumentProcessorConfig = {
|
|
|
2286
2337
|
* vertex_ai) can be switched without code changes.
|
|
2287
2338
|
*/
|
|
2288
2339
|
model?: string;
|
|
2340
|
+
/**
|
|
2341
|
+
* Maximum pages per OCR request for the "mistral" processor.
|
|
2342
|
+
* Vertex AI OCR rejects documents over 30 pages; the PDF is split into
|
|
2343
|
+
* chunks of this size and each chunk is OCR'd independently.
|
|
2344
|
+
* Defaults to 25 (safely under the Vertex AI 30-page limit).
|
|
2345
|
+
*/
|
|
2346
|
+
maxPagesPerChunk?: number;
|
|
2289
2347
|
};
|
|
2290
2348
|
/**
|
|
2291
2349
|
* Optional cost-attribution context, forwarded to LiteLLM as spend tags
|
|
@@ -2522,4 +2580,4 @@ declare const ExuluPython: {
|
|
|
2522
2580
|
instructions: typeof getPythonSetupInstructions;
|
|
2523
2581
|
};
|
|
2524
2582
|
|
|
2525
|
-
export { type ChunkerOperation, type ChunkerResponse, type JOB_STATUS as EXULU_JOB_STATUS, JOB_STATUS_ENUM as EXULU_JOB_STATUS_ENUM, type STATISTICS_TYPE as EXULU_STATISTICS_TYPE, STATISTICS_TYPE_ENUM as EXULU_STATISTICS_TYPE_ENUM, type ExuluAgent, ExuluApp, ExuluAuthentication, ExuluChunkers, ExuluContext, type ExuluContextEmbedder, ExuluDatabase, ExuluDefaultProviders, ExuluDefaultTools, ExuluDocumentProcessor, ExuluEval, type Item as ExuluItem, ExuluJobs, type ExuluOauthConfig, type ExuluOauthToolContext, ExuluOtel, ExuluProvider, ExuluPython, queues as ExuluQueues, ExuluReranker, ExuluTool, trajectoryRegistry as ExuluTrajectoryRegistry, ExuluVariables, defaultChunker };
|
|
2583
|
+
export { type ChunkerOperation, type ChunkerResponse, type JOB_STATUS as EXULU_JOB_STATUS, JOB_STATUS_ENUM as EXULU_JOB_STATUS_ENUM, type STATISTICS_TYPE as EXULU_STATISTICS_TYPE, STATISTICS_TYPE_ENUM as EXULU_STATISTICS_TYPE_ENUM, type ExuluAgent, ExuluApp, ExuluAuthentication, ExuluChunkers, ExuluContext, type ExuluContextEmbedder, ExuluDatabase, ExuluDefaultProviders, ExuluDefaultTools, ExuluDocumentProcessor, ExuluEval, type Item as ExuluItem, ExuluJobs, type ExuluOauthConfig, type ExuluOauthToolContext, ExuluOtel, ExuluProvider, ExuluPython, queues as ExuluQueues, ExuluReadApi, ExuluReranker, ExuluTool, trajectoryRegistry as ExuluTrajectoryRegistry, ExuluVariables, type VectorSearchChunkResult, defaultChunker, postgresClient };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as _opentelemetry_sdk_node from '@opentelemetry/sdk-node';
|
|
2
2
|
import * as knex from 'knex';
|
|
3
|
+
import { Knex } from 'knex';
|
|
3
4
|
import { RedisClientType } from 'redis';
|
|
4
5
|
import * as bullmq from 'bullmq';
|
|
5
6
|
import { Queue } from 'bullmq';
|
|
@@ -148,6 +149,7 @@ interface ExuluAgent {
|
|
|
148
149
|
instructions?: string;
|
|
149
150
|
feedback?: boolean;
|
|
150
151
|
suggestions_enabled?: boolean;
|
|
152
|
+
sandbox_enabled?: boolean;
|
|
151
153
|
slug?: string;
|
|
152
154
|
tools?: {
|
|
153
155
|
id: string;
|
|
@@ -786,6 +788,9 @@ declare class ExuluContext {
|
|
|
786
788
|
after?: number;
|
|
787
789
|
};
|
|
788
790
|
entityFilter?: EntityFilter;
|
|
791
|
+
/** Precomputed query embedding; forwarded to vectorSearch to skip re-embedding (caller must use
|
|
792
|
+
* this context's embedding model). Spread into vectorSearch via `...options` below. */
|
|
793
|
+
queryEmbedding?: number[];
|
|
789
794
|
}) => Promise<{
|
|
790
795
|
itemFilters: SearchFilters;
|
|
791
796
|
chunkFilters: SearchFilters;
|
|
@@ -828,9 +833,11 @@ declare class ExuluContext {
|
|
|
828
833
|
getItem: ({ item }: {
|
|
829
834
|
item: Item;
|
|
830
835
|
}) => Promise<Item>;
|
|
831
|
-
getItems: ({ filters, fields, }: {
|
|
836
|
+
getItems: ({ filters, fields, user, role, }: {
|
|
832
837
|
filters?: any[];
|
|
833
838
|
fields?: string[];
|
|
839
|
+
user?: User;
|
|
840
|
+
role?: string;
|
|
834
841
|
}) => Promise<Item[]>;
|
|
835
842
|
embeddings: {
|
|
836
843
|
generate: {
|
|
@@ -1804,6 +1811,50 @@ declare class RecursiveChunker extends BaseChunker {
|
|
|
1804
1811
|
toString(): string;
|
|
1805
1812
|
}
|
|
1806
1813
|
|
|
1814
|
+
interface AuthorizedReadOpts {
|
|
1815
|
+
itemIds?: string[];
|
|
1816
|
+
externalIds?: string[];
|
|
1817
|
+
chunkIndexRange?: {
|
|
1818
|
+
from?: number;
|
|
1819
|
+
to?: number;
|
|
1820
|
+
};
|
|
1821
|
+
}
|
|
1822
|
+
/**
|
|
1823
|
+
* Supported, RBAC-safe read API for retrieval clients (e.g. the agentic harness).
|
|
1824
|
+
* Relevance + visibility still go through ExuluContext.search(); this namespace
|
|
1825
|
+
* adds the read shapes search() cannot express, always over authorized rows.
|
|
1826
|
+
*/
|
|
1827
|
+
declare const ExuluReadApi: {
|
|
1828
|
+
getTableName: (id: string) => string;
|
|
1829
|
+
getChunksTableName: (id: string) => string;
|
|
1830
|
+
getEntitiesTableName: (id: string) => string;
|
|
1831
|
+
getChunkEntitiesTableName: (id: string) => string;
|
|
1832
|
+
entitiesAvailable: (context: {
|
|
1833
|
+
id: string;
|
|
1834
|
+
entities?: unknown;
|
|
1835
|
+
}) => Promise<boolean>;
|
|
1836
|
+
authorizedRead: (context: {
|
|
1837
|
+
id: string;
|
|
1838
|
+
fields?: unknown[];
|
|
1839
|
+
entities?: unknown;
|
|
1840
|
+
}, user: User, role: string, opts?: AuthorizedReadOpts) => Promise<VectorSearchChunkResult[]>;
|
|
1841
|
+
embedQuery: (context: {
|
|
1842
|
+
id: string;
|
|
1843
|
+
name?: string;
|
|
1844
|
+
embedder: {
|
|
1845
|
+
model: string;
|
|
1846
|
+
};
|
|
1847
|
+
}, text: string, opts?: {
|
|
1848
|
+
user?: User;
|
|
1849
|
+
role?: string;
|
|
1850
|
+
inputType?: "document" | "query";
|
|
1851
|
+
}) => Promise<number[]>;
|
|
1852
|
+
};
|
|
1853
|
+
|
|
1854
|
+
declare function postgresClient(): Promise<{
|
|
1855
|
+
db: Knex;
|
|
1856
|
+
}>;
|
|
1857
|
+
|
|
1807
1858
|
/**
|
|
1808
1859
|
* Represents the essential data for a sentence within a text.
|
|
1809
1860
|
*
|
|
@@ -2286,6 +2337,13 @@ type DocumentProcessorConfig = {
|
|
|
2286
2337
|
* vertex_ai) can be switched without code changes.
|
|
2287
2338
|
*/
|
|
2288
2339
|
model?: string;
|
|
2340
|
+
/**
|
|
2341
|
+
* Maximum pages per OCR request for the "mistral" processor.
|
|
2342
|
+
* Vertex AI OCR rejects documents over 30 pages; the PDF is split into
|
|
2343
|
+
* chunks of this size and each chunk is OCR'd independently.
|
|
2344
|
+
* Defaults to 25 (safely under the Vertex AI 30-page limit).
|
|
2345
|
+
*/
|
|
2346
|
+
maxPagesPerChunk?: number;
|
|
2289
2347
|
};
|
|
2290
2348
|
/**
|
|
2291
2349
|
* Optional cost-attribution context, forwarded to LiteLLM as spend tags
|
|
@@ -2522,4 +2580,4 @@ declare const ExuluPython: {
|
|
|
2522
2580
|
instructions: typeof getPythonSetupInstructions;
|
|
2523
2581
|
};
|
|
2524
2582
|
|
|
2525
|
-
export { type ChunkerOperation, type ChunkerResponse, type JOB_STATUS as EXULU_JOB_STATUS, JOB_STATUS_ENUM as EXULU_JOB_STATUS_ENUM, type STATISTICS_TYPE as EXULU_STATISTICS_TYPE, STATISTICS_TYPE_ENUM as EXULU_STATISTICS_TYPE_ENUM, type ExuluAgent, ExuluApp, ExuluAuthentication, ExuluChunkers, ExuluContext, type ExuluContextEmbedder, ExuluDatabase, ExuluDefaultProviders, ExuluDefaultTools, ExuluDocumentProcessor, ExuluEval, type Item as ExuluItem, ExuluJobs, type ExuluOauthConfig, type ExuluOauthToolContext, ExuluOtel, ExuluProvider, ExuluPython, queues as ExuluQueues, ExuluReranker, ExuluTool, trajectoryRegistry as ExuluTrajectoryRegistry, ExuluVariables, defaultChunker };
|
|
2583
|
+
export { type ChunkerOperation, type ChunkerResponse, type JOB_STATUS as EXULU_JOB_STATUS, JOB_STATUS_ENUM as EXULU_JOB_STATUS_ENUM, type STATISTICS_TYPE as EXULU_STATISTICS_TYPE, STATISTICS_TYPE_ENUM as EXULU_STATISTICS_TYPE_ENUM, type ExuluAgent, ExuluApp, ExuluAuthentication, ExuluChunkers, ExuluContext, type ExuluContextEmbedder, ExuluDatabase, ExuluDefaultProviders, ExuluDefaultTools, ExuluDocumentProcessor, ExuluEval, type Item as ExuluItem, ExuluJobs, type ExuluOauthConfig, type ExuluOauthToolContext, ExuluOtel, ExuluProvider, ExuluPython, queues as ExuluQueues, ExuluReadApi, ExuluReranker, ExuluTool, trajectoryRegistry as ExuluTrajectoryRegistry, ExuluVariables, type VectorSearchChunkResult, defaultChunker, postgresClient };
|