@zokizuan/satori-core 1.2.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/core/context.d.ts +4 -1
- package/dist/core/context.js +76 -27
- package/dist/splitter/ast-splitter.js +1 -1
- package/dist/splitter/langchain-splitter.js +2 -3
- package/dist/types.d.ts +11 -0
- package/dist/utils/env-manager.js +1 -1
- package/dist/vectordb/index.d.ts +2 -1
- package/dist/vectordb/index.js +4 -1
- package/dist/vectordb/milvus-restful-vectordb.d.ts +2 -2
- package/dist/vectordb/milvus-restful-vectordb.js +4 -3
- package/dist/vectordb/milvus-vectordb.js +2 -6
- package/dist/vectordb/remote-delete.d.ts +20 -0
- package/dist/vectordb/remote-delete.js +60 -0
- package/dist/vectordb/types.d.ts +8 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -36,7 +36,13 @@ const context = new Context({
|
|
|
36
36
|
});
|
|
37
37
|
|
|
38
38
|
await context.indexCodebase('/absolute/path/to/repo');
|
|
39
|
-
const results = await context.semanticSearch(
|
|
39
|
+
const results = await context.semanticSearch({
|
|
40
|
+
codebasePath: '/absolute/path/to/repo',
|
|
41
|
+
query: 'authentication logic',
|
|
42
|
+
topK: 5,
|
|
43
|
+
retrievalMode: 'hybrid',
|
|
44
|
+
scorePolicy: { kind: 'topk_only' }
|
|
45
|
+
});
|
|
40
46
|
```
|
|
41
47
|
|
|
42
48
|
## Development
|
package/dist/core/context.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Splitter } from '../splitter';
|
|
2
2
|
import { Embedding } from '../embedding';
|
|
3
3
|
import { VectorDatabase, IndexCompletionMarkerDocument } from '../vectordb';
|
|
4
|
-
import { SemanticSearchResult } from '../types';
|
|
4
|
+
import { SemanticSearchRequest, SemanticSearchResult } from '../types';
|
|
5
5
|
import { FileSynchronizer } from '../sync/synchronizer';
|
|
6
6
|
export interface ContextConfig {
|
|
7
7
|
embedding?: Embedding;
|
|
@@ -125,7 +125,10 @@ export declare class Context {
|
|
|
125
125
|
* @param topK Number of results to return
|
|
126
126
|
* @param threshold Similarity threshold
|
|
127
127
|
*/
|
|
128
|
+
semanticSearch(request: SemanticSearchRequest): Promise<SemanticSearchResult[]>;
|
|
128
129
|
semanticSearch(codebasePath: string, query: string, topK?: number, threshold?: number, filterExpr?: string): Promise<SemanticSearchResult[]>;
|
|
130
|
+
private normalizeSemanticSearchRequest;
|
|
131
|
+
private resolveSemanticSearchRequest;
|
|
129
132
|
private buildSemanticSearchFilterExpr;
|
|
130
133
|
private queryCompletionMarkerRows;
|
|
131
134
|
clearIndexCompletionMarker(codebasePath: string): Promise<void>;
|
package/dist/core/context.js
CHANGED
|
@@ -353,18 +353,14 @@ class Context {
|
|
|
353
353
|
}
|
|
354
354
|
}
|
|
355
355
|
}
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
* @param threshold Similarity threshold
|
|
362
|
-
*/
|
|
363
|
-
async semanticSearch(codebasePath, query, topK = 5, threshold = 0.5, filterExpr) {
|
|
364
|
-
const isHybrid = this.getIsHybrid();
|
|
356
|
+
async semanticSearch(requestOrCodebasePath, query, topK = 5, threshold = 0.5, filterExpr) {
|
|
357
|
+
const request = this.normalizeSemanticSearchRequest(requestOrCodebasePath, query, topK, threshold, filterExpr);
|
|
358
|
+
const resolvedRequest = this.resolveSemanticSearchRequest(request);
|
|
359
|
+
const codebasePath = resolvedRequest.codebasePath;
|
|
360
|
+
const isHybrid = resolvedRequest.retrievalMode !== 'dense' && this.getIsHybrid() === true;
|
|
365
361
|
const searchType = isHybrid === true ? 'hybrid search' : 'semantic search';
|
|
366
|
-
console.log(`[Context] 🔍 Executing ${searchType}: "${query}" in ${codebasePath}`);
|
|
367
|
-
const effectiveFilterExpr = this.buildSemanticSearchFilterExpr(filterExpr);
|
|
362
|
+
console.log(`[Context] 🔍 Executing ${searchType}: "${resolvedRequest.query}" in ${codebasePath}`);
|
|
363
|
+
const effectiveFilterExpr = this.buildSemanticSearchFilterExpr(resolvedRequest.filterExpr);
|
|
368
364
|
const normalizeBreadcrumbs = (value) => {
|
|
369
365
|
if (!Array.isArray(value)) {
|
|
370
366
|
return undefined;
|
|
@@ -387,15 +383,15 @@ class Context {
|
|
|
387
383
|
if (isHybrid === true) {
|
|
388
384
|
try {
|
|
389
385
|
// Check collection stats to see if it has data
|
|
390
|
-
|
|
386
|
+
await this.vectorDatabase.query(collectionName, '', ['id'], 1);
|
|
391
387
|
console.log(`[Context] 🔍 Collection '${collectionName}' exists and appears to have data`);
|
|
392
388
|
}
|
|
393
389
|
catch (error) {
|
|
394
390
|
console.log(`[Context] ⚠️ Collection '${collectionName}' exists but may be empty or not properly indexed:`, error);
|
|
395
391
|
}
|
|
396
392
|
// 1. Generate query vector
|
|
397
|
-
console.log(`[Context] 🔍 Generating embeddings for query: "${query}"`);
|
|
398
|
-
const queryEmbedding = await this.embedding.embed(query);
|
|
393
|
+
console.log(`[Context] 🔍 Generating embeddings for query: "${resolvedRequest.query}"`);
|
|
394
|
+
const queryEmbedding = await this.embedding.embed(resolvedRequest.query);
|
|
399
395
|
console.log(`[Context] ✅ Generated embedding vector with dimension: ${queryEmbedding.vector.length}`);
|
|
400
396
|
console.log(`[Context] 🔍 First 5 embedding values: [${queryEmbedding.vector.slice(0, 5).join(', ')}]`);
|
|
401
397
|
// 2. Prepare hybrid search requests
|
|
@@ -404,17 +400,17 @@ class Context {
|
|
|
404
400
|
data: queryEmbedding.vector,
|
|
405
401
|
anns_field: "vector",
|
|
406
402
|
param: { "nprobe": 10 },
|
|
407
|
-
limit: topK
|
|
403
|
+
limit: resolvedRequest.topK
|
|
408
404
|
},
|
|
409
405
|
{
|
|
410
|
-
data: query,
|
|
406
|
+
data: resolvedRequest.query,
|
|
411
407
|
anns_field: "sparse_vector",
|
|
412
408
|
param: { "drop_ratio_search": 0.2 },
|
|
413
|
-
limit: topK
|
|
409
|
+
limit: resolvedRequest.topK
|
|
414
410
|
}
|
|
415
411
|
];
|
|
416
412
|
console.log(`[Context] 🔍 Search request 1 (dense): anns_field="${searchRequests[0].anns_field}", vector_dim=${queryEmbedding.vector.length}, limit=${searchRequests[0].limit}`);
|
|
417
|
-
console.log(`[Context] 🔍 Search request 2 (sparse): anns_field="${searchRequests[1].anns_field}", query_text="${query}", limit=${searchRequests[1].limit}`);
|
|
413
|
+
console.log(`[Context] 🔍 Search request 2 (sparse): anns_field="${searchRequests[1].anns_field}", query_text="${resolvedRequest.query}", limit=${searchRequests[1].limit}`);
|
|
418
414
|
// 3. Execute hybrid search
|
|
419
415
|
console.log(`[Context] 🔍 Executing hybrid search with RRF reranking...`);
|
|
420
416
|
const searchResults = await this.vectorDatabase.hybridSearch(collectionName, searchRequests, {
|
|
@@ -422,7 +418,7 @@ class Context {
|
|
|
422
418
|
strategy: 'rrf',
|
|
423
419
|
params: { k: 100 }
|
|
424
420
|
},
|
|
425
|
-
limit: topK,
|
|
421
|
+
limit: resolvedRequest.topK,
|
|
426
422
|
// Hybrid RRF scores are backend/rerank relative, so dense similarity
|
|
427
423
|
// thresholds can erase valid sparse lexical matches before MCP ranking.
|
|
428
424
|
filterExpr: effectiveFilterExpr
|
|
@@ -439,7 +435,9 @@ class Context {
|
|
|
439
435
|
breadcrumbs: normalizeBreadcrumbs(result.document.metadata.breadcrumbs),
|
|
440
436
|
indexedAt: typeof result.document.metadata.indexedAt === 'string' ? result.document.metadata.indexedAt : undefined,
|
|
441
437
|
symbolId: typeof result.document.metadata.symbolId === 'string' ? result.document.metadata.symbolId : undefined,
|
|
442
|
-
symbolLabel: typeof result.document.metadata.symbolLabel === 'string' ? result.document.metadata.symbolLabel : undefined
|
|
438
|
+
symbolLabel: typeof result.document.metadata.symbolLabel === 'string' ? result.document.metadata.symbolLabel : undefined,
|
|
439
|
+
backendScore: result.score,
|
|
440
|
+
backendScoreKind: 'rrf_fusion'
|
|
443
441
|
}));
|
|
444
442
|
console.log(`[Context] ✅ Found ${results.length} relevant hybrid results`);
|
|
445
443
|
if (results.length > 0) {
|
|
@@ -450,9 +448,12 @@ class Context {
|
|
|
450
448
|
else {
|
|
451
449
|
// Regular semantic search
|
|
452
450
|
// 1. Generate query vector
|
|
453
|
-
const queryEmbedding = await this.embedding.embed(query);
|
|
451
|
+
const queryEmbedding = await this.embedding.embed(resolvedRequest.query);
|
|
452
|
+
const denseThreshold = resolvedRequest.scorePolicy.kind === 'dense_similarity_min'
|
|
453
|
+
? resolvedRequest.scorePolicy.min
|
|
454
|
+
: undefined;
|
|
454
455
|
// 2. Search in vector database
|
|
455
|
-
const searchResults = await this.vectorDatabase.search(collectionName, queryEmbedding.vector, { topK, threshold, filterExpr: effectiveFilterExpr });
|
|
456
|
+
const searchResults = await this.vectorDatabase.search(collectionName, queryEmbedding.vector, { topK: resolvedRequest.topK, threshold: denseThreshold, filterExpr: effectiveFilterExpr });
|
|
456
457
|
// 3. Convert to semantic search result format
|
|
457
458
|
const results = searchResults.map(result => ({
|
|
458
459
|
content: result.document.content,
|
|
@@ -464,12 +465,54 @@ class Context {
|
|
|
464
465
|
breadcrumbs: normalizeBreadcrumbs(result.document.metadata.breadcrumbs),
|
|
465
466
|
indexedAt: typeof result.document.metadata.indexedAt === 'string' ? result.document.metadata.indexedAt : undefined,
|
|
466
467
|
symbolId: typeof result.document.metadata.symbolId === 'string' ? result.document.metadata.symbolId : undefined,
|
|
467
|
-
symbolLabel: typeof result.document.metadata.symbolLabel === 'string' ? result.document.metadata.symbolLabel : undefined
|
|
468
|
+
symbolLabel: typeof result.document.metadata.symbolLabel === 'string' ? result.document.metadata.symbolLabel : undefined,
|
|
469
|
+
backendScore: result.score,
|
|
470
|
+
backendScoreKind: 'dense_similarity'
|
|
468
471
|
}));
|
|
469
472
|
console.log(`[Context] ✅ Found ${results.length} relevant results`);
|
|
470
473
|
return results;
|
|
471
474
|
}
|
|
472
475
|
}
|
|
476
|
+
normalizeSemanticSearchRequest(requestOrCodebasePath, query, topK = 5, threshold = 0.5, filterExpr) {
|
|
477
|
+
if (typeof requestOrCodebasePath === 'string') {
|
|
478
|
+
return {
|
|
479
|
+
codebasePath: requestOrCodebasePath,
|
|
480
|
+
query: query ?? '',
|
|
481
|
+
topK,
|
|
482
|
+
filterExpr,
|
|
483
|
+
...(threshold > 0
|
|
484
|
+
? {
|
|
485
|
+
retrievalMode: 'dense',
|
|
486
|
+
scorePolicy: { kind: 'dense_similarity_min', min: threshold }
|
|
487
|
+
}
|
|
488
|
+
: {
|
|
489
|
+
scorePolicy: { kind: 'topk_only' }
|
|
490
|
+
})
|
|
491
|
+
};
|
|
492
|
+
}
|
|
493
|
+
return requestOrCodebasePath;
|
|
494
|
+
}
|
|
495
|
+
resolveSemanticSearchRequest(request) {
|
|
496
|
+
const hybridEnabled = this.getIsHybrid() === true;
|
|
497
|
+
const retrievalMode = request.retrievalMode ?? (hybridEnabled ? 'hybrid' : 'dense');
|
|
498
|
+
const scorePolicy = request.scorePolicy ?? (retrievalMode === 'dense'
|
|
499
|
+
? { kind: 'dense_similarity_min', min: 0.5 }
|
|
500
|
+
: { kind: 'topk_only' });
|
|
501
|
+
if (request.retrievalMode !== undefined && retrievalMode !== 'dense' && hybridEnabled !== true) {
|
|
502
|
+
throw new Error(`${retrievalMode} retrieval requires hybrid search support, but HYBRID_MODE is disabled.`);
|
|
503
|
+
}
|
|
504
|
+
if (retrievalMode !== 'dense' && scorePolicy.kind === 'dense_similarity_min') {
|
|
505
|
+
throw new Error(`Dense similarity threshold score policy is invalid for ${retrievalMode} retrieval.`);
|
|
506
|
+
}
|
|
507
|
+
return {
|
|
508
|
+
codebasePath: request.codebasePath,
|
|
509
|
+
query: request.query,
|
|
510
|
+
topK: request.topK ?? 5,
|
|
511
|
+
retrievalMode,
|
|
512
|
+
filterExpr: request.filterExpr ?? '',
|
|
513
|
+
scorePolicy
|
|
514
|
+
};
|
|
515
|
+
}
|
|
473
516
|
buildSemanticSearchFilterExpr(filterExpr) {
|
|
474
517
|
const markerExclusion = `fileExtension != "${vectordb_1.INDEX_COMPLETION_MARKER_FILE_EXTENSION}"`;
|
|
475
518
|
if (!filterExpr || filterExpr.trim().length === 0) {
|
|
@@ -588,7 +631,7 @@ class Context {
|
|
|
588
631
|
const collectionExists = await this.vectorDatabase.hasCollection(collectionName);
|
|
589
632
|
progressCallback?.({ phase: 'Removing index data...', current: 50, total: 100, percentage: 50 });
|
|
590
633
|
if (collectionExists) {
|
|
591
|
-
await this.vectorDatabase
|
|
634
|
+
await (0, vectordb_1.deleteCollectionWithVerification)(this.vectorDatabase, collectionName);
|
|
592
635
|
}
|
|
593
636
|
// Delete snapshot file
|
|
594
637
|
await synchronizer_1.FileSynchronizer.deleteSnapshot(codebasePath);
|
|
@@ -948,7 +991,10 @@ class Context {
|
|
|
948
991
|
}
|
|
949
992
|
const relativePath = path.relative(codebasePath, chunk.metadata.filePath);
|
|
950
993
|
const fileExtension = path.extname(chunk.metadata.filePath);
|
|
951
|
-
const { filePath, startLine, endLine, ...restMetadata } = chunk.metadata;
|
|
994
|
+
const { filePath: omittedFilePath, startLine: omittedStartLine, endLine: omittedEndLine, ...restMetadata } = chunk.metadata;
|
|
995
|
+
void omittedFilePath;
|
|
996
|
+
void omittedStartLine;
|
|
997
|
+
void omittedEndLine;
|
|
952
998
|
return {
|
|
953
999
|
id: this.generateId(relativePath, chunk.metadata.startLine || 0, chunk.metadata.endLine || 0, chunk.content),
|
|
954
1000
|
content: chunk.content, // Full text content for BM25 and storage
|
|
@@ -977,7 +1023,10 @@ class Context {
|
|
|
977
1023
|
}
|
|
978
1024
|
const relativePath = path.relative(codebasePath, chunk.metadata.filePath);
|
|
979
1025
|
const fileExtension = path.extname(chunk.metadata.filePath);
|
|
980
|
-
const { filePath, startLine, endLine, ...restMetadata } = chunk.metadata;
|
|
1026
|
+
const { filePath: omittedFilePath, startLine: omittedStartLine, endLine: omittedEndLine, ...restMetadata } = chunk.metadata;
|
|
1027
|
+
void omittedFilePath;
|
|
1028
|
+
void omittedStartLine;
|
|
1029
|
+
void omittedEndLine;
|
|
981
1030
|
return {
|
|
982
1031
|
id: this.generateId(relativePath, chunk.metadata.startLine || 0, chunk.metadata.endLine || 0, chunk.content),
|
|
983
1032
|
vector: embeddings[index].vector,
|
|
@@ -1114,7 +1163,7 @@ class Context {
|
|
|
1114
1163
|
return [];
|
|
1115
1164
|
}
|
|
1116
1165
|
}
|
|
1117
|
-
catch
|
|
1166
|
+
catch {
|
|
1118
1167
|
if (fileName.includes('global')) {
|
|
1119
1168
|
console.log(`📄 No ${fileName} file found`);
|
|
1120
1169
|
}
|
|
@@ -156,7 +156,7 @@ class AstCodeSplitter {
|
|
|
156
156
|
}
|
|
157
157
|
return this.addOverlap(refinedChunks);
|
|
158
158
|
}
|
|
159
|
-
splitLargeChunk(chunk,
|
|
159
|
+
splitLargeChunk(chunk, _originalCode) {
|
|
160
160
|
const lines = chunk.content.split('\n');
|
|
161
161
|
const subChunks = [];
|
|
162
162
|
let currentChunk = '';
|
|
@@ -23,7 +23,7 @@ class LangChainCodeSplitter {
|
|
|
23
23
|
// Split code
|
|
24
24
|
const documents = await splitter.createDocuments([code]);
|
|
25
25
|
// Convert to CodeChunk format
|
|
26
|
-
return documents.map((doc
|
|
26
|
+
return documents.map((doc) => {
|
|
27
27
|
const lines = doc.metadata?.loc?.lines || { from: 1, to: 1 };
|
|
28
28
|
return {
|
|
29
29
|
content: doc.pageContent,
|
|
@@ -86,7 +86,7 @@ class LangChainCodeSplitter {
|
|
|
86
86
|
chunkOverlap: this.chunkOverlap,
|
|
87
87
|
});
|
|
88
88
|
const documents = await splitter.createDocuments([code]);
|
|
89
|
-
return documents.map((doc
|
|
89
|
+
return documents.map((doc) => {
|
|
90
90
|
const lines = this.estimateLines(doc.pageContent, code);
|
|
91
91
|
return {
|
|
92
92
|
content: doc.pageContent,
|
|
@@ -101,7 +101,6 @@ class LangChainCodeSplitter {
|
|
|
101
101
|
}
|
|
102
102
|
estimateLines(chunk, originalCode) {
|
|
103
103
|
// Simple line number estimation
|
|
104
|
-
const codeLines = originalCode.split('\n');
|
|
105
104
|
const chunkLines = chunk.split('\n');
|
|
106
105
|
// Find chunk position in original code
|
|
107
106
|
const chunkStart = originalCode.indexOf(chunk);
|
package/dist/types.d.ts
CHANGED
|
@@ -1,8 +1,17 @@
|
|
|
1
|
+
import type { BackendScoreKind, RetrievalMode, ScorePolicy } from './vectordb/types';
|
|
1
2
|
export interface SearchQuery {
|
|
2
3
|
term: string;
|
|
3
4
|
includeContent?: boolean;
|
|
4
5
|
limit?: number;
|
|
5
6
|
}
|
|
7
|
+
export interface SemanticSearchRequest {
|
|
8
|
+
codebasePath: string;
|
|
9
|
+
query: string;
|
|
10
|
+
topK?: number;
|
|
11
|
+
retrievalMode?: RetrievalMode;
|
|
12
|
+
filterExpr?: string;
|
|
13
|
+
scorePolicy?: ScorePolicy;
|
|
14
|
+
}
|
|
6
15
|
export interface SemanticSearchResult {
|
|
7
16
|
content: string;
|
|
8
17
|
relativePath: string;
|
|
@@ -14,5 +23,7 @@ export interface SemanticSearchResult {
|
|
|
14
23
|
indexedAt?: string;
|
|
15
24
|
symbolId?: string;
|
|
16
25
|
symbolLabel?: string;
|
|
26
|
+
backendScore?: number;
|
|
27
|
+
backendScoreKind?: BackendScoreKind;
|
|
17
28
|
}
|
|
18
29
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/vectordb/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
export { VectorDocument, SearchOptions, VectorSearchResult, VectorDatabase, CollectionDetails, VectorStoreBackendInfo, HybridSearchRequest, HybridSearchOptions, HybridSearchResult, RerankStrategy, IndexCompletionFingerprint, IndexCompletionMarkerDocument, INDEX_COMPLETION_MARKER_DOC_ID, INDEX_COMPLETION_MARKER_FILE_EXTENSION, INDEX_COMPLETION_MARKER_RELATIVE_PATH, COLLECTION_LIMIT_MESSAGE } from './types';
|
|
1
|
+
export { VectorDocument, SearchOptions, VectorSearchResult, VectorDatabase, CollectionDetails, VectorStoreBackendInfo, HybridSearchRequest, HybridSearchOptions, HybridSearchResult, RerankStrategy, RetrievalMode, ScorePolicy, BackendScoreKind, IndexCompletionFingerprint, IndexCompletionMarkerDocument, INDEX_COMPLETION_MARKER_DOC_ID, INDEX_COMPLETION_MARKER_FILE_EXTENSION, INDEX_COMPLETION_MARKER_RELATIVE_PATH, COLLECTION_LIMIT_MESSAGE } from './types';
|
|
2
2
|
export { MilvusRestfulVectorDatabase, MilvusRestfulConfig } from './milvus-restful-vectordb';
|
|
3
3
|
export { MilvusVectorDatabase, MilvusConfig } from './milvus-vectordb';
|
|
4
|
+
export { RemoteCollectionDeletePendingError, deleteCollectionWithVerification, VerifiedCollectionDeleteOptions, VerifiedCollectionDeleteResult } from './remote-delete';
|
|
4
5
|
export { ClusterManager, ZillizConfig, Project, Cluster, CreateFreeClusterRequest, CreateFreeClusterResponse, CreateFreeClusterWithDetailsResponse, DescribeClusterResponse } from './zilliz-utils';
|
|
5
6
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/vectordb/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ClusterManager = exports.MilvusVectorDatabase = exports.MilvusRestfulVectorDatabase = exports.COLLECTION_LIMIT_MESSAGE = exports.INDEX_COMPLETION_MARKER_RELATIVE_PATH = exports.INDEX_COMPLETION_MARKER_FILE_EXTENSION = exports.INDEX_COMPLETION_MARKER_DOC_ID = void 0;
|
|
3
|
+
exports.ClusterManager = exports.deleteCollectionWithVerification = exports.RemoteCollectionDeletePendingError = exports.MilvusVectorDatabase = exports.MilvusRestfulVectorDatabase = exports.COLLECTION_LIMIT_MESSAGE = exports.INDEX_COMPLETION_MARKER_RELATIVE_PATH = exports.INDEX_COMPLETION_MARKER_FILE_EXTENSION = exports.INDEX_COMPLETION_MARKER_DOC_ID = void 0;
|
|
4
4
|
// Re-export types and interfaces
|
|
5
5
|
var types_1 = require("./types");
|
|
6
6
|
Object.defineProperty(exports, "INDEX_COMPLETION_MARKER_DOC_ID", { enumerable: true, get: function () { return types_1.INDEX_COMPLETION_MARKER_DOC_ID; } });
|
|
@@ -12,6 +12,9 @@ var milvus_restful_vectordb_1 = require("./milvus-restful-vectordb");
|
|
|
12
12
|
Object.defineProperty(exports, "MilvusRestfulVectorDatabase", { enumerable: true, get: function () { return milvus_restful_vectordb_1.MilvusRestfulVectorDatabase; } });
|
|
13
13
|
var milvus_vectordb_1 = require("./milvus-vectordb");
|
|
14
14
|
Object.defineProperty(exports, "MilvusVectorDatabase", { enumerable: true, get: function () { return milvus_vectordb_1.MilvusVectorDatabase; } });
|
|
15
|
+
var remote_delete_1 = require("./remote-delete");
|
|
16
|
+
Object.defineProperty(exports, "RemoteCollectionDeletePendingError", { enumerable: true, get: function () { return remote_delete_1.RemoteCollectionDeletePendingError; } });
|
|
17
|
+
Object.defineProperty(exports, "deleteCollectionWithVerification", { enumerable: true, get: function () { return remote_delete_1.deleteCollectionWithVerification; } });
|
|
15
18
|
var zilliz_utils_1 = require("./zilliz-utils");
|
|
16
19
|
Object.defineProperty(exports, "ClusterManager", { enumerable: true, get: function () { return zilliz_utils_1.ClusterManager; } });
|
|
17
20
|
//# sourceMappingURL=index.js.map
|
|
@@ -47,7 +47,7 @@ export declare class MilvusRestfulVectorDatabase implements VectorDatabase {
|
|
|
47
47
|
* Make HTTP request to Milvus REST API
|
|
48
48
|
*/
|
|
49
49
|
private makeRequest;
|
|
50
|
-
createCollection(collectionName: string, dimension: number,
|
|
50
|
+
createCollection(collectionName: string, dimension: number, _description?: string): Promise<void>;
|
|
51
51
|
/**
|
|
52
52
|
* Create index for vector field using the Index Create API
|
|
53
53
|
*/
|
|
@@ -65,7 +65,7 @@ export declare class MilvusRestfulVectorDatabase implements VectorDatabase {
|
|
|
65
65
|
search(collectionName: string, queryVector: number[], options?: SearchOptions): Promise<VectorSearchResult[]>;
|
|
66
66
|
delete(collectionName: string, ids: string[]): Promise<void>;
|
|
67
67
|
query(collectionName: string, filter: string, outputFields: string[], limit?: number): Promise<Record<string, any>[]>;
|
|
68
|
-
createHybridCollection(collectionName: string, dimension: number,
|
|
68
|
+
createHybridCollection(collectionName: string, dimension: number, _description?: string): Promise<void>;
|
|
69
69
|
private createHybridIndexes;
|
|
70
70
|
insertHybrid(collectionName: string, documents: VectorDocument[]): Promise<void>;
|
|
71
71
|
hybridSearch(collectionName: string, searchRequests: HybridSearchRequest[], options?: HybridSearchOptions): Promise<HybridSearchResult[]>;
|
|
@@ -13,6 +13,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
13
13
|
exports.MilvusRestfulVectorDatabase = void 0;
|
|
14
14
|
const types_1 = require("./types");
|
|
15
15
|
const zilliz_utils_1 = require("./zilliz-utils");
|
|
16
|
+
const remote_delete_1 = require("./remote-delete");
|
|
16
17
|
function normalizeHost(address) {
|
|
17
18
|
const withProtocol = address.includes('://') ? address : `http://${address}`;
|
|
18
19
|
try {
|
|
@@ -161,7 +162,7 @@ class MilvusRestfulVectorDatabase {
|
|
|
161
162
|
throw error;
|
|
162
163
|
}
|
|
163
164
|
}
|
|
164
|
-
async createCollection(collectionName, dimension,
|
|
165
|
+
async createCollection(collectionName, dimension, _description) {
|
|
165
166
|
await this.ensureInitialized();
|
|
166
167
|
try {
|
|
167
168
|
const restfulConfig = this.config;
|
|
@@ -495,7 +496,7 @@ class MilvusRestfulVectorDatabase {
|
|
|
495
496
|
throw error;
|
|
496
497
|
}
|
|
497
498
|
}
|
|
498
|
-
async createHybridCollection(collectionName, dimension,
|
|
499
|
+
async createHybridCollection(collectionName, dimension, _description) {
|
|
499
500
|
try {
|
|
500
501
|
const restfulConfig = this.config;
|
|
501
502
|
const collectionSchema = {
|
|
@@ -772,7 +773,7 @@ class MilvusRestfulVectorDatabase {
|
|
|
772
773
|
};
|
|
773
774
|
try {
|
|
774
775
|
await createCollectionWithLimitCheck(this.makeRequest.bind(this), collectionSchema);
|
|
775
|
-
await
|
|
776
|
+
await (0, remote_delete_1.deleteCollectionWithVerification)(this, collectionName);
|
|
776
777
|
return true;
|
|
777
778
|
}
|
|
778
779
|
catch (error) {
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.MilvusVectorDatabase = void 0;
|
|
4
4
|
const milvus2_sdk_node_1 = require("@zilliz/milvus2-sdk-node");
|
|
5
5
|
const zilliz_utils_1 = require("./zilliz-utils");
|
|
6
|
+
const remote_delete_1 = require("./remote-delete");
|
|
6
7
|
const COLLECTION_LIMIT_PATTERNS = [
|
|
7
8
|
/exceeded the limit number of collections/i,
|
|
8
9
|
/collection limit/i,
|
|
@@ -753,12 +754,7 @@ class MilvusVectorDatabase {
|
|
|
753
754
|
};
|
|
754
755
|
try {
|
|
755
756
|
await this.client.createCollection(createCollectionParams);
|
|
756
|
-
|
|
757
|
-
if (await this.client.hasCollection({ collection_name: collectionName })) {
|
|
758
|
-
await this.client.dropCollection({
|
|
759
|
-
collection_name: collectionName,
|
|
760
|
-
});
|
|
761
|
-
}
|
|
757
|
+
await (0, remote_delete_1.deleteCollectionWithVerification)(this, collectionName);
|
|
762
758
|
return true;
|
|
763
759
|
}
|
|
764
760
|
catch (error) {
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { VectorDatabase } from './types';
|
|
2
|
+
export interface VerifiedCollectionDeleteOptions {
|
|
3
|
+
maxAttempts?: number;
|
|
4
|
+
initialBackoffMs?: number;
|
|
5
|
+
backoffMultiplier?: number;
|
|
6
|
+
sleep?: (ms: number) => Promise<void>;
|
|
7
|
+
}
|
|
8
|
+
export interface VerifiedCollectionDeleteResult {
|
|
9
|
+
collectionName: string;
|
|
10
|
+
attempts: number;
|
|
11
|
+
verifiedAbsent: boolean;
|
|
12
|
+
}
|
|
13
|
+
export declare class RemoteCollectionDeletePendingError extends Error {
|
|
14
|
+
readonly collectionName: string;
|
|
15
|
+
readonly attempts: number;
|
|
16
|
+
readonly lastError?: unknown;
|
|
17
|
+
constructor(collectionName: string, attempts: number, lastError?: unknown);
|
|
18
|
+
}
|
|
19
|
+
export declare function deleteCollectionWithVerification(vectorDatabase: Pick<VectorDatabase, 'dropCollection' | 'hasCollection'>, collectionName: string, options?: VerifiedCollectionDeleteOptions): Promise<VerifiedCollectionDeleteResult>;
|
|
20
|
+
//# sourceMappingURL=remote-delete.d.ts.map
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RemoteCollectionDeletePendingError = void 0;
|
|
4
|
+
exports.deleteCollectionWithVerification = deleteCollectionWithVerification;
|
|
5
|
+
const DEFAULT_MAX_ATTEMPTS = 5;
|
|
6
|
+
const DEFAULT_INITIAL_BACKOFF_MS = 100;
|
|
7
|
+
const DEFAULT_BACKOFF_MULTIPLIER = 2;
|
|
8
|
+
function defaultSleep(ms) {
|
|
9
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
10
|
+
}
|
|
11
|
+
function formatError(error) {
|
|
12
|
+
return error instanceof Error ? error.message : String(error);
|
|
13
|
+
}
|
|
14
|
+
class RemoteCollectionDeletePendingError extends Error {
|
|
15
|
+
constructor(collectionName, attempts, lastError) {
|
|
16
|
+
const detail = lastError ? ` Last error: ${formatError(lastError)}` : '';
|
|
17
|
+
super(`Remote collection deletion did not complete for '${collectionName}' after ${attempts} attempt(s).${detail}`);
|
|
18
|
+
this.name = 'RemoteCollectionDeletePendingError';
|
|
19
|
+
this.collectionName = collectionName;
|
|
20
|
+
this.attempts = attempts;
|
|
21
|
+
this.lastError = lastError;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
exports.RemoteCollectionDeletePendingError = RemoteCollectionDeletePendingError;
|
|
25
|
+
async function deleteCollectionWithVerification(vectorDatabase, collectionName, options = {}) {
|
|
26
|
+
const maxAttempts = Math.max(1, Math.floor(options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS));
|
|
27
|
+
const initialBackoffMs = Math.max(0, options.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS);
|
|
28
|
+
const backoffMultiplier = Math.max(1, options.backoffMultiplier ?? DEFAULT_BACKOFF_MULTIPLIER);
|
|
29
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
30
|
+
if (!await vectorDatabase.hasCollection(collectionName)) {
|
|
31
|
+
return { collectionName, attempts: 0, verifiedAbsent: true };
|
|
32
|
+
}
|
|
33
|
+
let lastError;
|
|
34
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
35
|
+
lastError = undefined;
|
|
36
|
+
try {
|
|
37
|
+
await vectorDatabase.dropCollection(collectionName);
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
lastError = error;
|
|
41
|
+
}
|
|
42
|
+
try {
|
|
43
|
+
const stillExists = await vectorDatabase.hasCollection(collectionName);
|
|
44
|
+
if (!stillExists) {
|
|
45
|
+
return { collectionName, attempts: attempt, verifiedAbsent: true };
|
|
46
|
+
}
|
|
47
|
+
if (!lastError) {
|
|
48
|
+
lastError = new Error(`dropCollection returned successfully but '${collectionName}' still exists.`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
lastError = error;
|
|
53
|
+
}
|
|
54
|
+
if (attempt < maxAttempts && initialBackoffMs > 0) {
|
|
55
|
+
await sleep(initialBackoffMs * Math.pow(backoffMultiplier, attempt - 1));
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
throw new RemoteCollectionDeletePendingError(collectionName, maxAttempts, lastError);
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=remote-delete.js.map
|
package/dist/vectordb/types.d.ts
CHANGED
|
@@ -8,6 +8,14 @@ export interface VectorDocument {
|
|
|
8
8
|
fileExtension: string;
|
|
9
9
|
metadata: Record<string, any>;
|
|
10
10
|
}
|
|
11
|
+
export type RetrievalMode = 'dense' | 'lexical' | 'hybrid';
|
|
12
|
+
export type ScorePolicy = {
|
|
13
|
+
kind: 'dense_similarity_min';
|
|
14
|
+
min: number;
|
|
15
|
+
} | {
|
|
16
|
+
kind: 'topk_only';
|
|
17
|
+
};
|
|
18
|
+
export type BackendScoreKind = 'dense_similarity' | 'lexical_rank' | 'rrf_fusion';
|
|
11
19
|
export interface SearchOptions {
|
|
12
20
|
topK?: number;
|
|
13
21
|
filter?: Record<string, any>;
|