@igoruehara/canvas-flow 0.1.0

package/server/rag/rag-service.d.ts

@@ -0,0 +1,361 @@
+import { OnModuleInit } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { HttpBatchService } from '../http-batch/http-batch-service';
+import { MemoryService } from '../memory/memory-service';
+import { ProviderConfigService } from '../provider-config/provider-config-service';
+import { DocumentsService } from '../documents/documents-service';
+interface RagDocumentInput {
+    text: string;
+    embeddingName?: string;
+    embeddingId?: string;
+    agentId?: string;
+    extraFields?: Record<string, any>;
+}
+interface UploadedRagFile {
+    buffer: Buffer;
+    originalname?: string;
+    mimetype?: string;
+    size?: number;
+}
+export declare class RagService implements OnModuleInit {
+    private readonly configService;
+    private readonly memoryService;
+    private readonly httpBatchService;
+    private readonly providerConfigService;
+    private readonly documentsService?;
+    private readonly logger;
+    private openAIClient?;
+    private openAIClientError?;
+    private milvusClient?;
+    private azureBlobContainer?;
+    private runtimeSettings?;
+    private openAIRuntimeConfig?;
+    private providerSignature;
+    private azureSearchFieldCache;
+    constructor(configService: ConfigService, memoryService: MemoryService, httpBatchService: HttpBatchService, providerConfigService: ProviderConfigService, documentsService?: DocumentsService);
+    private applyProviderSettings;
+    private refreshProviderSettings;
+    onModuleInit(): Promise<void>;
+    private getDefaultCollectionName;
+    private getEmbeddingModel;
+    private getChatModel;
+    private getOcrModel;
+    private getEmbeddingDimensions;
+    private normalizeOpenAIProvider;
+    private getOpenAIClientForProvider;
+    private getEmbeddingModelForProvider;
+    private getChatModelForProvider;
+    private getAzureSearchEndpoint;
+    private getAzureSearchApiKey;
+    private getAzureSearchIndexName;
+    private getAzureSearchApiVersion;
+    private isAzureSearchConfigured;
+    private collectAzureSearchFieldSchema;
+    private getAzureSearchIndexSchema;
+    private getAzureSearchIndexFields;
+    private azureSearchSelectFields;
+    private azureSearchVectorField;
+    private azureSearchTextFields;
+    private pickAzureSearchDocumentField;
+    private azureSearchSemanticConfiguration;
+    private withTransientRetry;
+    private fetchWithRetry;
+    private chunkArray;
+    private contentHash;
+    private ensureVectorDimensions;
+    private shouldUseAzureSearch;
+    private shouldUseMilvusSearch;
+    private shouldUseAzureBlob;
+    private parseBoolean;
+    private parseJsonField;
+    private escapeMilvusString;
+    private isBinaryDecodedText;
+    private buildDataUrl;
+    private sanitizeText;
+    private sanitizeObject;
+    private chunkText;
+    private normalizeMilvusResults;
+    private normalizeMilvusData;
+    private getChunkIndex;
+    private stripChunkFields;
+    private streamToString;
+    private normalizeBlobSearchTerms;
+    private blobMatchesTerms;
+    private detectTextOverlap;
+    private joinChunkTexts;
+    private buildAgentExpr;
+    private buildDocumentExpr;
+    private groupDocumentRows;
+    private queryRows;
+    private flushCollection;
+    private mergeHybridResults;
+    private clampSearchInt;
+    private clampSearchFloat;
+    private getSearchScore;
+    private parseExtraFields;
+    private checkRelevanceFlag;
+    private applyRelevanceBoost;
+    private combineSearchResultsV2;
+    private buildExtraFieldsExpr;
+    private formatMilvusValue;
+    private mergeExtraFieldsFilters;
+    private isEmptyRoundFilter;
+    private getExtraField;
+    private toComparableMetadataValue;
+    private applyMetadataOrdering;
+    createCollection(collectionName?: string): Promise<import("@zilliz/milvus2-sdk-node").ResStatus>;
+    createIndex(collectionName?: string): Promise<{
+        collectionName: string;
+        dense: any;
+        sparse: any;
+        azureSearch: any;
+    }>;
+    embeddingCreate(text: string, provider?: string, model?: string): Promise<any>;
+    embeddingCreateBatch(texts: string[], provider?: string, model?: string): Promise<any[]>;
+    private toAzureSafeId;
+    private escapeODataString;
+    private isAzureFilterable;
+    private pushAzureFilter;
+    private buildAzureFilter;
+    private buildAzureRagMetadata;
+    private buildAzureDocumentMetadata;
+    private buildAzureBlobChunkPayload;
+    private normalizeAzureBlobJsonPayload;
+    private uploadRowsToAzureBlob;
+    uploadTextToAzureBlob(blobName: string, content: string, contentType?: string): Promise<{
+        blobName: string;
+        blobUrl: string;
+        contentType: string;
+        bytes: number;
+    }>;
+    uploadChunksToAzureBlob(params: {
+        collectionName?: string;
+        text: string;
+        embeddingName?: string;
+        embeddingId?: string;
+        agentId?: string;
+        extraFields?: Record<string, any>;
+        chunkSize?: number;
+        chunkOverlap?: number;
+        embeddingProvider?: string;
+        embeddingModel?: string;
+        embeddingBatchSize?: number;
+    }): Promise<{
+        collectionName: string;
+        embeddingName: any;
+        embeddingId: string;
+        chunks: number;
+        blobs: {
+            id: string;
+            blobName: string;
+            blobUrl: string;
+        }[];
+    }>;
+    listAzureBlobDocuments(prefix?: string, options?: any): Promise<{
+        prefix: string;
+        filters: {
+            query: string;
+            contentType: string;
+            modifiedAfter: any;
+            modifiedBefore: any;
+            minBytes: number;
+            maxBytes: number;
+            maxTextBytes: number;
+        };
+        debug: {
+            scanned: number;
+            skippedByContentType: number;
+            skippedBySize: number;
+            skippedByDate: number;
+            skippedByQuery: number;
+            textDownloads: number;
+            textDownloadErrors: number;
+            textSkippedBySize: number;
+        };
+        total: number;
+        blobs: any[];
+    }>;
+    readAzureBlobDocument(blobName: string): Promise<{
+        blobName: string;
+        blobUrl: string;
+        contentType: string;
+        size: number;
+        lastModified: string | Date;
+        metadata: any;
+        text: string;
+        rawText: string;
+        payload: any;
+        hasVector: boolean;
+        vectorDimensions: number;
+    }>;
+    private indexRowsInAzureSearch;
+    private createAzureSearchIndex;
+    private searchAzureSearch;
+    addDocuments(collectionName: string | undefined, documents: RagDocumentInput[], options?: any): Promise<{
+        success: boolean;
+        inserted: number;
+        totalChunks: number;
+        batches: number;
+        message: string;
+        response?: undefined;
+        responses?: undefined;
+        azureSearch?: undefined;
+        azureBlobs?: undefined;
+    } | {
+        success: boolean;
+        inserted: number;
+        totalChunks: number;
+        batches: number;
+        response: any;
+        responses: any[];
+        azureSearch: {
+            indexed: number;
+            batches: number;
+            vectorField: string;
+            textFields: string[];
+            expectedDimensions: number;
+            responses: any[];
+        };
+        azureBlobs: number;
+        message: string;
+    }>;
+    private formatWorksheetCell;
+    private extractXlsxText;
+    private extractPdfText;
+    private extractXlsxStructure;
+    private stripHtml;
+    private extractPlaceholders;
+    private textSections;
+    private extractFileStructure;
+    private extractTextDirect;
+    private extractTextWithOpenAI;
+    private extractTextFromUploadedFile;
+    extractFiles(files: UploadedRagFile[], body?: any): Promise<{
+        files: any[];
+        documents: number;
+    }>;
+    addDocumentsFromFiles(files: UploadedRagFile[], body?: any): Promise<{
+        collectionName: any;
+        files: any[];
+        documents: number;
+        success: boolean;
+        inserted: number;
+        totalChunks: number;
+        batches: number;
+        message: string;
+        response?: undefined;
+        responses?: undefined;
+        azureSearch?: undefined;
+        azureBlobs?: undefined;
+    } | {
+        collectionName: any;
+        files: any[];
+        documents: number;
+        success: boolean;
+        inserted: number;
+        totalChunks: number;
+        batches: number;
+        response: any;
+        responses: any[];
+        azureSearch: {
+            indexed: number;
+            batches: number;
+            vectorField: string;
+            textFields: string[];
+            expectedDimensions: number;
+            responses: any[];
+        };
+        azureBlobs: number;
+        message: string;
+    }>;
+    listDocuments(collectionName?: string, agentId?: string, query?: string, options?: any): Promise<{
+        collectionName: string;
+        total: number;
+        documents: {
+            id: any;
+            embeddingId: any;
+            embeddingName: any;
+            agentId: any;
+            extraFields: {
+                [x: string]: any;
+            };
+            chunksCount: number;
+            ids: any[];
+            text: any;
+            textLength: any;
+            textPreview: any;
+        }[];
+    }>;
+    getDocument(collectionName: string | undefined, idOrEmbeddingId: string, agentId?: string): Promise<{
+        collectionName: string;
+        document: {
+            id: any;
+            embeddingId: any;
+            embeddingName: any;
+            agentId: any;
+            extraFields: {
+                [x: string]: any;
+            };
+            chunksCount: number;
+            ids: any[];
+            text: any;
+            textLength: any;
+            textPreview: any;
+        };
+    }>;
+    deleteDocument(collectionName: string | undefined, idOrEmbeddingId: string, agentId?: string): Promise<{
+        collectionName: string;
+        deleted: boolean;
+        filter: string;
+        response: import("@zilliz/milvus2-sdk-node").MutationResult;
+    }>;
+    updateDocument(collectionName: string | undefined, idOrEmbeddingId: string, payload?: any): Promise<{
+        success: boolean;
+        inserted: number;
+        totalChunks: number;
+        batches: number;
+        message: string;
+        response?: undefined;
+        responses?: undefined;
+        azureSearch?: undefined;
+        azureBlobs?: undefined;
+        collectionName: string;
+        embeddingId: any;
+        updated: boolean;
+    } | {
+        success: boolean;
+        inserted: number;
+        totalChunks: number;
+        batches: number;
+        response: any;
+        responses: any[];
+        azureSearch: {
+            indexed: number;
+            batches: number;
+            vectorField: string;
+            textFields: string[];
+            expectedDimensions: number;
+            responses: any[];
+        };
+        azureBlobs: number;
+        message: string;
+        collectionName: string;
+        embeddingId: any;
+        updated: boolean;
+    }>;
+    listCollections(): Promise<import("@zilliz/milvus2-sdk-node").CollectionData[]>;
+    private combineProviderSearchResults;
+    searchHybrid(query: string, collectionName: string | undefined, agentId: string | undefined, params?: any): Promise<any>;
+    private buildHttpBatchTool;
+    private toOpenAIHistory;
+    private extractTextFromCompletion;
+    chatLlmRag(text: string, agentId: string | undefined, params?: any): Promise<{
+        text: any;
+        conversationId: any;
+        docs: any;
+        searchDebug: any;
+        trace: any[];
+        model: string;
+    }>;
+}
+export {};