@cmdoss/memwal-sdk 0.6.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +547 -547
- package/BENCHMARKS.md +238 -238
- package/README.md +310 -181
- package/dist/ai-sdk/tools.d.ts +2 -2
- package/dist/ai-sdk/tools.js +2 -2
- package/dist/client/ClientMemoryManager.js +2 -2
- package/dist/client/ClientMemoryManager.js.map +1 -1
- package/dist/client/PersonalDataWallet.d.ts.map +1 -1
- package/dist/client/SimplePDWClient.d.ts +29 -1
- package/dist/client/SimplePDWClient.d.ts.map +1 -1
- package/dist/client/SimplePDWClient.js +45 -13
- package/dist/client/SimplePDWClient.js.map +1 -1
- package/dist/client/namespaces/EmbeddingsNamespace.d.ts +1 -1
- package/dist/client/namespaces/EmbeddingsNamespace.js +1 -1
- package/dist/client/namespaces/MemoryNamespace.d.ts +31 -0
- package/dist/client/namespaces/MemoryNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/MemoryNamespace.js +272 -39
- package/dist/client/namespaces/MemoryNamespace.js.map +1 -1
- package/dist/client/namespaces/consolidated/AINamespace.d.ts +2 -2
- package/dist/client/namespaces/consolidated/AINamespace.js +2 -2
- package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts +12 -2
- package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/consolidated/BlockchainNamespace.js +62 -4
- package/dist/client/namespaces/consolidated/BlockchainNamespace.js.map +1 -1
- package/dist/client/namespaces/consolidated/StorageNamespace.d.ts +67 -2
- package/dist/client/namespaces/consolidated/StorageNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/consolidated/StorageNamespace.js +549 -16
- package/dist/client/namespaces/consolidated/StorageNamespace.js.map +1 -1
- package/dist/config/ConfigurationHelper.js +61 -61
- package/dist/config/defaults.js +2 -2
- package/dist/config/defaults.js.map +1 -1
- package/dist/graph/GraphService.js +21 -21
- package/dist/graph/GraphService.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/infrastructure/seal/EncryptionService.d.ts +9 -5
- package/dist/infrastructure/seal/EncryptionService.d.ts.map +1 -1
- package/dist/infrastructure/seal/EncryptionService.js +37 -15
- package/dist/infrastructure/seal/EncryptionService.js.map +1 -1
- package/dist/infrastructure/seal/SealService.d.ts +13 -5
- package/dist/infrastructure/seal/SealService.d.ts.map +1 -1
- package/dist/infrastructure/seal/SealService.js +36 -34
- package/dist/infrastructure/seal/SealService.js.map +1 -1
- package/dist/langchain/createPDWRAG.js +30 -30
- package/dist/retrieval/MemoryDecryptionPipeline.d.ts.map +1 -1
- package/dist/retrieval/MemoryDecryptionPipeline.js +2 -1
- package/dist/retrieval/MemoryDecryptionPipeline.js.map +1 -1
- package/dist/retrieval/MemoryRetrievalService.d.ts +31 -0
- package/dist/retrieval/MemoryRetrievalService.d.ts.map +1 -1
- package/dist/retrieval/MemoryRetrievalService.js +44 -4
- package/dist/retrieval/MemoryRetrievalService.js.map +1 -1
- package/dist/services/CapabilityService.d.ts.map +1 -1
- package/dist/services/CapabilityService.js +30 -14
- package/dist/services/CapabilityService.js.map +1 -1
- package/dist/services/CrossContextPermissionService.d.ts.map +1 -1
- package/dist/services/CrossContextPermissionService.js +9 -7
- package/dist/services/CrossContextPermissionService.js.map +1 -1
- package/dist/services/EmbeddingService.d.ts +28 -1
- package/dist/services/EmbeddingService.d.ts.map +1 -1
- package/dist/services/EmbeddingService.js +54 -0
- package/dist/services/EmbeddingService.js.map +1 -1
- package/dist/services/EncryptionService.d.ts.map +1 -1
- package/dist/services/EncryptionService.js +6 -5
- package/dist/services/EncryptionService.js.map +1 -1
- package/dist/services/GeminiAIService.js +309 -309
- package/dist/services/IndexManager.d.ts +5 -1
- package/dist/services/IndexManager.d.ts.map +1 -1
- package/dist/services/IndexManager.js +17 -40
- package/dist/services/IndexManager.js.map +1 -1
- package/dist/services/QueryService.js +1 -1
- package/dist/services/QueryService.js.map +1 -1
- package/dist/services/StorageService.d.ts +11 -0
- package/dist/services/StorageService.d.ts.map +1 -1
- package/dist/services/StorageService.js +73 -10
- package/dist/services/StorageService.js.map +1 -1
- package/dist/services/TransactionService.d.ts +20 -0
- package/dist/services/TransactionService.d.ts.map +1 -1
- package/dist/services/TransactionService.js +43 -0
- package/dist/services/TransactionService.js.map +1 -1
- package/dist/services/ViewService.js +2 -2
- package/dist/services/ViewService.js.map +1 -1
- package/dist/services/storage/QuiltBatchManager.d.ts +101 -1
- package/dist/services/storage/QuiltBatchManager.d.ts.map +1 -1
- package/dist/services/storage/QuiltBatchManager.js +410 -20
- package/dist/services/storage/QuiltBatchManager.js.map +1 -1
- package/dist/services/storage/index.d.ts +1 -1
- package/dist/services/storage/index.d.ts.map +1 -1
- package/dist/services/storage/index.js.map +1 -1
- package/dist/utils/LRUCache.d.ts +106 -0
- package/dist/utils/LRUCache.d.ts.map +1 -0
- package/dist/utils/LRUCache.js +281 -0
- package/dist/utils/LRUCache.js.map +1 -0
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +2 -0
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/memoryIndexOnChain.d.ts +212 -0
- package/dist/utils/memoryIndexOnChain.d.ts.map +1 -0
- package/dist/utils/memoryIndexOnChain.js +312 -0
- package/dist/utils/memoryIndexOnChain.js.map +1 -0
- package/dist/utils/rebuildIndexNode.d.ts +29 -0
- package/dist/utils/rebuildIndexNode.d.ts.map +1 -1
- package/dist/utils/rebuildIndexNode.js +366 -98
- package/dist/utils/rebuildIndexNode.js.map +1 -1
- package/dist/vector/HnswWasmService.d.ts +20 -5
- package/dist/vector/HnswWasmService.d.ts.map +1 -1
- package/dist/vector/HnswWasmService.js +73 -40
- package/dist/vector/HnswWasmService.js.map +1 -1
- package/dist/vector/IHnswService.d.ts +10 -1
- package/dist/vector/IHnswService.d.ts.map +1 -1
- package/dist/vector/IHnswService.js.map +1 -1
- package/dist/vector/NodeHnswService.d.ts +16 -0
- package/dist/vector/NodeHnswService.d.ts.map +1 -1
- package/dist/vector/NodeHnswService.js +84 -5
- package/dist/vector/NodeHnswService.js.map +1 -1
- package/dist/vector/createHnswService.d.ts +1 -1
- package/dist/vector/createHnswService.js +1 -1
- package/dist/vector/index.d.ts +1 -1
- package/dist/vector/index.js +1 -1
- package/package.json +157 -157
- package/src/access/PermissionService.ts +635 -635
- package/src/aggregation/AggregationService.ts +389 -389
- package/src/ai-sdk/PDWVectorStore.ts +715 -715
- package/src/ai-sdk/index.ts +65 -65
- package/src/ai-sdk/tools.ts +460 -460
- package/src/ai-sdk/types.ts +404 -404
- package/src/batch/BatchManager.ts +597 -597
- package/src/batch/BatchingService.ts +429 -429
- package/src/batch/MemoryProcessingCache.ts +492 -492
- package/src/batch/index.ts +30 -30
- package/src/browser.ts +200 -200
- package/src/client/ClientMemoryManager.ts +987 -987
- package/src/client/PersonalDataWallet.ts +345 -345
- package/src/client/SimplePDWClient.ts +1289 -1222
- package/src/client/factory.ts +154 -154
- package/src/client/namespaces/AnalyticsNamespace.ts +377 -377
- package/src/client/namespaces/BatchNamespace.ts +356 -356
- package/src/client/namespaces/CacheNamespace.ts +123 -123
- package/src/client/namespaces/CapabilityNamespace.ts +217 -217
- package/src/client/namespaces/ClassifyNamespace.ts +169 -169
- package/src/client/namespaces/ContextNamespace.ts +297 -297
- package/src/client/namespaces/EmbeddingsNamespace.ts +99 -99
- package/src/client/namespaces/EncryptionNamespace.ts +221 -221
- package/src/client/namespaces/GraphNamespace.ts +468 -468
- package/src/client/namespaces/IndexNamespace.ts +361 -361
- package/src/client/namespaces/MemoryNamespace.ts +1422 -1135
- package/src/client/namespaces/PermissionsNamespace.ts +254 -254
- package/src/client/namespaces/PipelineNamespace.ts +220 -220
- package/src/client/namespaces/SearchNamespace.ts +1049 -1049
- package/src/client/namespaces/StorageNamespace.ts +458 -458
- package/src/client/namespaces/TxNamespace.ts +260 -260
- package/src/client/namespaces/WalletNamespace.ts +243 -243
- package/src/client/namespaces/consolidated/AINamespace.ts +449 -449
- package/src/client/namespaces/consolidated/BlockchainNamespace.ts +607 -546
- package/src/client/namespaces/consolidated/SecurityNamespace.ts +648 -648
- package/src/client/namespaces/consolidated/StorageNamespace.ts +1141 -497
- package/src/client/namespaces/consolidated/index.ts +39 -39
- package/src/client/signers/KeypairSigner.ts +108 -108
- package/src/client/signers/UnifiedSigner.ts +110 -110
- package/src/client/signers/WalletAdapterSigner.ts +159 -159
- package/src/client/signers/index.ts +26 -26
- package/src/config/ConfigurationHelper.ts +412 -412
- package/src/config/defaults.ts +51 -51
- package/src/config/index.ts +8 -8
- package/src/config/validation.ts +70 -70
- package/src/core/index.ts +14 -14
- package/src/core/interfaces/IService.ts +307 -307
- package/src/core/interfaces/index.ts +8 -8
- package/src/core/types/capability.ts +297 -297
- package/src/core/types/index.ts +870 -870
- package/src/core/types/wallet.ts +270 -270
- package/src/core/types.ts +9 -9
- package/src/core/wallet.ts +222 -222
- package/src/embedding/index.ts +19 -19
- package/src/embedding/types.ts +357 -357
- package/src/errors/index.ts +602 -602
- package/src/errors/recovery.ts +461 -461
- package/src/errors/validation.ts +567 -567
- package/src/generated/pdw/capability.ts +319 -319
- package/src/graph/GraphService.ts +887 -887
- package/src/graph/KnowledgeGraphManager.ts +728 -728
- package/src/graph/index.ts +25 -25
- package/src/index.ts +498 -474
- package/src/infrastructure/index.ts +22 -22
- package/src/infrastructure/seal/EncryptionService.ts +628 -603
- package/src/infrastructure/seal/SealService.ts +613 -615
- package/src/infrastructure/seal/index.ts +9 -9
- package/src/infrastructure/sui/BlockchainManager.ts +627 -627
- package/src/infrastructure/sui/SuiService.ts +888 -888
- package/src/infrastructure/sui/index.ts +9 -9
- package/src/infrastructure/walrus/StorageManager.ts +604 -604
- package/src/infrastructure/walrus/WalrusStorageService.ts +612 -612
- package/src/infrastructure/walrus/index.ts +9 -9
- package/src/langchain/PDWEmbeddings.ts +145 -145
- package/src/langchain/PDWVectorStore.ts +456 -456
- package/src/langchain/createPDWRAG.ts +303 -303
- package/src/langchain/index.ts +47 -47
- package/src/permissions/ConsentRepository.browser.ts +249 -249
- package/src/permissions/ConsentRepository.ts +364 -364
- package/src/pipeline/MemoryPipeline.ts +862 -862
- package/src/pipeline/PipelineManager.ts +683 -683
- package/src/pipeline/index.ts +26 -26
- package/src/retrieval/AdvancedSearchService.ts +629 -629
- package/src/retrieval/MemoryAnalyticsService.ts +711 -711
- package/src/retrieval/MemoryDecryptionPipeline.ts +825 -824
- package/src/retrieval/MemoryRetrievalService.ts +904 -830
- package/src/retrieval/index.ts +42 -42
- package/src/services/BatchService.ts +352 -352
- package/src/services/CapabilityService.ts +464 -448
- package/src/services/ClassifierService.ts +465 -465
- package/src/services/CrossContextPermissionService.ts +486 -484
- package/src/services/EmbeddingService.ts +771 -706
- package/src/services/EncryptionService.ts +712 -711
- package/src/services/GeminiAIService.ts +753 -753
- package/src/services/IndexManager.ts +977 -1004
- package/src/services/MemoryIndexService.ts +1003 -1003
- package/src/services/MemoryService.ts +369 -369
- package/src/services/QueryService.ts +890 -890
- package/src/services/StorageService.ts +1182 -1111
- package/src/services/TransactionService.ts +838 -790
- package/src/services/VectorService.ts +462 -462
- package/src/services/ViewService.ts +484 -484
- package/src/services/index.ts +25 -25
- package/src/services/storage/BlobAttributesManager.ts +333 -333
- package/src/services/storage/KnowledgeGraphManager.ts +425 -425
- package/src/services/storage/MemorySearchManager.ts +387 -387
- package/src/services/storage/QuiltBatchManager.ts +1130 -660
- package/src/services/storage/WalrusMetadataManager.ts +268 -268
- package/src/services/storage/WalrusStorageManager.ts +287 -287
- package/src/services/storage/index.ts +57 -52
- package/src/types/index.ts +13 -13
- package/src/utils/LRUCache.ts +378 -0
- package/src/utils/index.ts +76 -68
- package/src/utils/memoryIndexOnChain.ts +507 -0
- package/src/utils/rebuildIndex.ts +290 -290
- package/src/utils/rebuildIndexNode.ts +771 -424
- package/src/vector/BrowserHnswIndexService.ts +758 -758
- package/src/vector/HnswWasmService.ts +731 -679
- package/src/vector/IHnswService.ts +233 -224
- package/src/vector/NodeHnswService.ts +833 -735
- package/src/vector/VectorManager.ts +478 -478
- package/src/vector/createHnswService.ts +135 -135
- package/src/vector/index.ts +56 -56
- package/src/wallet/ContextWalletService.ts +656 -656
- package/src/wallet/MainWalletService.ts +317 -317
|
@@ -1,456 +1,456 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* PDWVectorStore - LangChain VectorStore implementation for Personal Data Wallet
|
|
3
|
-
*
|
|
4
|
-
* Integrates PDW's decentralized storage (Walrus) and vector search (HNSW) with LangChain.
|
|
5
|
-
* Enables RAG workflows with memories stored on Sui blockchain and Walrus.
|
|
6
|
-
*
|
|
7
|
-
* Features:
|
|
8
|
-
* - Decentralized vector storage on Walrus
|
|
9
|
-
* - HNSW-powered similarity search (browser-compatible WebAssembly)
|
|
10
|
-
* - Blockchain-backed ownership via Sui
|
|
11
|
-
* - SEAL encryption for privacy (optional)
|
|
12
|
-
* - Memory metadata and filtering
|
|
13
|
-
*
|
|
14
|
-
* @example
|
|
15
|
-
* ```typescript
|
|
16
|
-
* import { PDWVectorStore, PDWEmbeddings } from 'personal-data-wallet-sdk/langchain';
|
|
17
|
-
* import { useCurrentAccount, useSuiClient } from '@mysten/dapp-kit';
|
|
18
|
-
*
|
|
19
|
-
* const embeddings = new PDWEmbeddings({ geminiApiKey });
|
|
20
|
-
* const vectorStore = new PDWVectorStore(embeddings, {
|
|
21
|
-
* userAddress: account.address,
|
|
22
|
-
* packageId: '0x...',
|
|
23
|
-
* walrusAggregator: 'https://aggregator.walrus-testnet.walrus.space',
|
|
24
|
-
* });
|
|
25
|
-
*
|
|
26
|
-
* // Add documents (requires Sui wallet signing)
|
|
27
|
-
* await vectorStore.addDocuments([
|
|
28
|
-
* { pageContent: 'Hello world', metadata: { category: 'greeting' } }
|
|
29
|
-
* ], { account, signAndExecute, client });
|
|
30
|
-
*
|
|
31
|
-
* // Search
|
|
32
|
-
* const results = await vectorStore.similaritySearch('hi', 5);
|
|
33
|
-
* ```
|
|
34
|
-
*/
|
|
35
|
-
|
|
36
|
-
import { VectorStore } from '@langchain/core/vectorstores';
|
|
37
|
-
import { Document } from '@langchain/core/documents';
|
|
38
|
-
import type { EmbeddingsInterface } from '@langchain/core/embeddings';
|
|
39
|
-
import type { Callbacks } from '@langchain/core/callbacks/manager';
|
|
40
|
-
import type { MaxMarginalRelevanceSearchOptions } from '@langchain/core/vectorstores';
|
|
41
|
-
import { ClientMemoryManager } from '../client/ClientMemoryManager';
|
|
42
|
-
import type { ClientMemoryManagerConfig } from '../client/ClientMemoryManager';
|
|
43
|
-
import { MemoryIndexService } from '../services/MemoryIndexService';
|
|
44
|
-
import type { MemorySearchQuery, MemorySearchResult } from '../services/MemoryIndexService';
|
|
45
|
-
import { StorageService } from '../services/StorageService';
|
|
46
|
-
import { EmbeddingService } from '../services/EmbeddingService';
|
|
47
|
-
import type { Transaction } from '@mysten/sui/transactions';
|
|
48
|
-
import type { SuiClient } from '@mysten/sui/client';
|
|
49
|
-
|
|
50
|
-
export interface PDWVectorStoreConfig {
|
|
51
|
-
// Required
|
|
52
|
-
userAddress: string;
|
|
53
|
-
|
|
54
|
-
// Sui Configuration
|
|
55
|
-
packageId: string;
|
|
56
|
-
accessRegistryId?: string;
|
|
57
|
-
network?: 'mainnet' | 'testnet' | 'devnet';
|
|
58
|
-
|
|
59
|
-
// Walrus Configuration
|
|
60
|
-
walrusPublisher?: string;
|
|
61
|
-
walrusAggregator?: string;
|
|
62
|
-
walrusNetwork?: 'testnet' | 'mainnet';
|
|
63
|
-
|
|
64
|
-
// AI Configuration (for MemoryIndexService)
|
|
65
|
-
geminiApiKey: string;
|
|
66
|
-
embeddingModel?: string;
|
|
67
|
-
embeddingDimensions?: number;
|
|
68
|
-
|
|
69
|
-
// Vector Index Configuration
|
|
70
|
-
maxElements?: number;
|
|
71
|
-
efConstruction?: number;
|
|
72
|
-
m?: number;
|
|
73
|
-
|
|
74
|
-
// Memory Configuration
|
|
75
|
-
defaultCategory?: string;
|
|
76
|
-
defaultImportance?: number;
|
|
77
|
-
encryptionEnabled?: boolean;
|
|
78
|
-
|
|
79
|
-
// SEAL Configuration
|
|
80
|
-
sealServerObjectIds?: string[];
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export interface PDWAddDocumentOptions {
|
|
84
|
-
// Sui wallet signing (required for adding documents)
|
|
85
|
-
account: { address: string };
|
|
86
|
-
signAndExecute: (params: { transaction: Transaction }, callbacks: {
|
|
87
|
-
onSuccess: (result: any) => void;
|
|
88
|
-
onError: (error: Error) => void;
|
|
89
|
-
}) => void;
|
|
90
|
-
client: SuiClient;
|
|
91
|
-
|
|
92
|
-
// Optional memory metadata
|
|
93
|
-
category?: string;
|
|
94
|
-
importance?: number;
|
|
95
|
-
tags?: string[];
|
|
96
|
-
encrypt?: boolean;
|
|
97
|
-
onProgress?: (status: string) => void;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* LangChain VectorStore implementation using Personal Data Wallet
|
|
102
|
-
*
|
|
103
|
-
* This class wraps PDW's ClientMemoryManager and MemoryIndexService to provide
|
|
104
|
-
* a standard LangChain VectorStore interface while preserving all PDW features:
|
|
105
|
-
* - Decentralized storage on Walrus
|
|
106
|
-
* - Blockchain ownership via Sui
|
|
107
|
-
* - SEAL encryption
|
|
108
|
-
* - HNSW vector search
|
|
109
|
-
*/
|
|
110
|
-
export class PDWVectorStore extends VectorStore {
|
|
111
|
-
private clientMemoryManager!: ClientMemoryManager;
|
|
112
|
-
private memoryIndexService!: MemoryIndexService;
|
|
113
|
-
private storageService!: StorageService;
|
|
114
|
-
private embeddingService!: EmbeddingService;
|
|
115
|
-
private config: PDWVectorStoreConfig;
|
|
116
|
-
|
|
117
|
-
// Track document to blobId mapping
|
|
118
|
-
private documentMapping = new Map<string, string>(); // blobId -> documentId
|
|
119
|
-
|
|
120
|
-
constructor(
|
|
121
|
-
embeddings: EmbeddingsInterface,
|
|
122
|
-
config: PDWVectorStoreConfig
|
|
123
|
-
) {
|
|
124
|
-
super(embeddings, {});
|
|
125
|
-
this.config = config;
|
|
126
|
-
|
|
127
|
-
// Initialize PDW services
|
|
128
|
-
this.initializeServices();
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
private initializeServices() {
|
|
132
|
-
// Initialize ClientMemoryManager
|
|
133
|
-
const managerConfig: ClientMemoryManagerConfig = {
|
|
134
|
-
packageId: this.config.packageId,
|
|
135
|
-
accessRegistryId: this.config.accessRegistryId || '',
|
|
136
|
-
walrusAggregator: this.config.walrusAggregator || 'https://aggregator.walrus-testnet.walrus.space',
|
|
137
|
-
geminiApiKey: this.config.geminiApiKey,
|
|
138
|
-
sealServerObjectIds: this.config.sealServerObjectIds,
|
|
139
|
-
walrusNetwork: this.config.walrusNetwork || 'testnet',
|
|
140
|
-
enableLocalIndexing: true,
|
|
141
|
-
};
|
|
142
|
-
|
|
143
|
-
this.clientMemoryManager = new ClientMemoryManager(managerConfig);
|
|
144
|
-
|
|
145
|
-
// Initialize services
|
|
146
|
-
this.embeddingService = new EmbeddingService({
|
|
147
|
-
apiKey: this.config.geminiApiKey,
|
|
148
|
-
model: this.config.embeddingModel || 'text-embedding-004',
|
|
149
|
-
dimensions: this.config.embeddingDimensions || 3072,
|
|
150
|
-
});
|
|
151
|
-
|
|
152
|
-
this.storageService = new StorageService({
|
|
153
|
-
walrusPublisherUrl: this.config.walrusPublisher,
|
|
154
|
-
walrusAggregatorUrl: this.config.walrusAggregator || 'https://aggregator.walrus-testnet.walrus.space',
|
|
155
|
-
network: (this.config.network === 'devnet' ? 'testnet' : this.config.network) || 'testnet',
|
|
156
|
-
});
|
|
157
|
-
|
|
158
|
-
this.memoryIndexService = new MemoryIndexService(
|
|
159
|
-
this.storageService,
|
|
160
|
-
{
|
|
161
|
-
maxElements: this.config.maxElements || 10000,
|
|
162
|
-
dimension: this.config.embeddingDimensions || 3072,
|
|
163
|
-
efConstruction: this.config.efConstruction || 200,
|
|
164
|
-
m: this.config.m || 16,
|
|
165
|
-
}
|
|
166
|
-
);
|
|
167
|
-
|
|
168
|
-
// Connect embedding service to index service
|
|
169
|
-
this.memoryIndexService.initialize(this.embeddingService, this.storageService);
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
/**
|
|
173
|
-
* Create PDWVectorStore from documents (LangChain factory pattern)
|
|
174
|
-
*
|
|
175
|
-
* @param docs - Documents to add
|
|
176
|
-
* @param embeddings - Embeddings interface
|
|
177
|
-
* @param dbConfig - Combined config object containing both PDWVectorStoreConfig and PDWAddDocumentOptions
|
|
178
|
-
*/
|
|
179
|
-
static async fromDocuments(
|
|
180
|
-
docs: Document[],
|
|
181
|
-
embeddings: EmbeddingsInterface,
|
|
182
|
-
dbConfig: PDWVectorStoreConfig & { addOptions?: PDWAddDocumentOptions }
|
|
183
|
-
): Promise<PDWVectorStore> {
|
|
184
|
-
const { addOptions, ...config } = dbConfig;
|
|
185
|
-
const instance = new PDWVectorStore(embeddings, config as PDWVectorStoreConfig);
|
|
186
|
-
|
|
187
|
-
if (addOptions) {
|
|
188
|
-
await instance.addDocuments(docs, addOptions);
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
return instance;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
/**
|
|
195
|
-
* Create PDWVectorStore from texts (LangChain factory pattern)
|
|
196
|
-
*
|
|
197
|
-
* @param texts - Text strings to add
|
|
198
|
-
* @param metadatas - Metadata for each text
|
|
199
|
-
* @param embeddings - Embeddings interface
|
|
200
|
-
* @param dbConfig - Combined config object containing both PDWVectorStoreConfig and PDWAddDocumentOptions
|
|
201
|
-
*/
|
|
202
|
-
static async fromTexts(
|
|
203
|
-
texts: string[],
|
|
204
|
-
metadatas: object[] | object,
|
|
205
|
-
embeddings: EmbeddingsInterface,
|
|
206
|
-
dbConfig: PDWVectorStoreConfig & { addOptions?: PDWAddDocumentOptions }
|
|
207
|
-
): Promise<PDWVectorStore> {
|
|
208
|
-
const docs = texts.map((text, i) => {
|
|
209
|
-
const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
|
|
210
|
-
return new Document({ pageContent: text, metadata });
|
|
211
|
-
});
|
|
212
|
-
return PDWVectorStore.fromDocuments(docs, embeddings, dbConfig);
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
/**
|
|
216
|
-
* Add documents to the vector store
|
|
217
|
-
*
|
|
218
|
-
* This creates memories on Sui blockchain and stores them on Walrus.
|
|
219
|
-
* Requires wallet signing via PDWAddDocumentOptions.
|
|
220
|
-
*/
|
|
221
|
-
async addDocuments(
|
|
222
|
-
documents: Document[],
|
|
223
|
-
options?: PDWAddDocumentOptions
|
|
224
|
-
): Promise<string[]> {
|
|
225
|
-
if (!options) {
|
|
226
|
-
throw new Error(
|
|
227
|
-
'PDWVectorStore.addDocuments requires options with wallet signing. ' +
|
|
228
|
-
'Pass { account, signAndExecute, client } to sign transactions.'
|
|
229
|
-
);
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
const blobIds: string[] = [];
|
|
233
|
-
|
|
234
|
-
for (const doc of documents) {
|
|
235
|
-
try {
|
|
236
|
-
if (options.onProgress) {
|
|
237
|
-
options.onProgress(`Creating memory: ${doc.pageContent.substring(0, 50)}...`);
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
// Create memory using ClientMemoryManager
|
|
241
|
-
// This handles: embedding → encryption → Walrus upload → on-chain registration
|
|
242
|
-
const blobId = await this.clientMemoryManager.createMemory({
|
|
243
|
-
content: doc.pageContent,
|
|
244
|
-
category: options.category || doc.metadata?.category || this.config.defaultCategory || 'general',
|
|
245
|
-
account: options.account,
|
|
246
|
-
signAndExecute: options.signAndExecute,
|
|
247
|
-
client: options.client,
|
|
248
|
-
onProgress: options.onProgress,
|
|
249
|
-
});
|
|
250
|
-
|
|
251
|
-
blobIds.push(blobId);
|
|
252
|
-
this.documentMapping.set(blobId, doc.id || blobId);
|
|
253
|
-
|
|
254
|
-
if (options.onProgress) {
|
|
255
|
-
options.onProgress(`✅ Memory created: ${blobId}`);
|
|
256
|
-
}
|
|
257
|
-
} catch (error) {
|
|
258
|
-
console.error(`Failed to add document:`, error);
|
|
259
|
-
throw new Error(
|
|
260
|
-
`Failed to add document: ${error instanceof Error ? error.message : String(error)}`
|
|
261
|
-
);
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
return blobIds;
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
/**
|
|
269
|
-
* Add vectors directly to the store
|
|
270
|
-
*
|
|
271
|
-
* This is useful when you already have embeddings and want to avoid regenerating them.
|
|
272
|
-
*/
|
|
273
|
-
async addVectors(
|
|
274
|
-
vectors: number[][],
|
|
275
|
-
documents: Document[],
|
|
276
|
-
options?: PDWAddDocumentOptions
|
|
277
|
-
): Promise<string[]> {
|
|
278
|
-
if (vectors.length !== documents.length) {
|
|
279
|
-
throw new Error('Vectors and documents must have the same length');
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
// Note: Currently ClientMemoryManager generates embeddings internally
|
|
283
|
-
// For now, we'll call addDocuments which will regenerate embeddings
|
|
284
|
-
// TODO: Add support for passing pre-computed embeddings to ClientMemoryManager
|
|
285
|
-
return this.addDocuments(documents, options);
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
/**
|
|
289
|
-
* Similarity search with score (core LangChain method)
|
|
290
|
-
*/
|
|
291
|
-
async similaritySearchVectorWithScore(
|
|
292
|
-
query: number[],
|
|
293
|
-
k: number,
|
|
294
|
-
filter?: Record<string, any>
|
|
295
|
-
): Promise<[Document, number][]> {
|
|
296
|
-
// Build search query for MemoryIndexService
|
|
297
|
-
const searchQuery: MemorySearchQuery = {
|
|
298
|
-
vector: query,
|
|
299
|
-
userAddress: this.config.userAddress,
|
|
300
|
-
k: k || 5,
|
|
301
|
-
threshold: filter?.minSimilarity || 0.0,
|
|
302
|
-
categories: filter?.category ? [filter.category] : filter?.categories,
|
|
303
|
-
tags: filter?.tags,
|
|
304
|
-
dateRange: filter?.dateRange,
|
|
305
|
-
importanceRange: filter?.importanceRange,
|
|
306
|
-
includeContent: false, // Don't fetch content by default (can be slow with decryption)
|
|
307
|
-
};
|
|
308
|
-
|
|
309
|
-
// Search using MemoryIndexService
|
|
310
|
-
const results = await this.memoryIndexService.searchMemories(searchQuery);
|
|
311
|
-
|
|
312
|
-
// Convert to LangChain format
|
|
313
|
-
return results.map((result: MemorySearchResult) => {
|
|
314
|
-
// Use metadata topic as pageContent if content not included
|
|
315
|
-
const pageContent = result.metadata?.topic || result.metadata?.category || '';
|
|
316
|
-
|
|
317
|
-
const doc = new Document({
|
|
318
|
-
pageContent,
|
|
319
|
-
metadata: {
|
|
320
|
-
...result.metadata,
|
|
321
|
-
blobId: result.blobId,
|
|
322
|
-
memoryId: result.memoryId,
|
|
323
|
-
similarity: result.similarity,
|
|
324
|
-
relevanceScore: result.relevanceScore,
|
|
325
|
-
},
|
|
326
|
-
});
|
|
327
|
-
|
|
328
|
-
return [doc, result.similarity] as [Document, number];
|
|
329
|
-
});
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
/**
|
|
333
|
-
* Similarity search (standard LangChain method)
|
|
334
|
-
*/
|
|
335
|
-
async similaritySearch(
|
|
336
|
-
query: string,
|
|
337
|
-
k: number = 5,
|
|
338
|
-
filter?: Record<string, any>
|
|
339
|
-
): Promise<Document[]> {
|
|
340
|
-
const resultsWithScore = await this.similaritySearchWithScore(query, k, filter);
|
|
341
|
-
return resultsWithScore.map(([doc]) => doc);
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
/**
|
|
345
|
-
* Similarity search with score using query string
|
|
346
|
-
*/
|
|
347
|
-
async similaritySearchWithScore(
|
|
348
|
-
query: string,
|
|
349
|
-
k: number = 5,
|
|
350
|
-
filter?: Record<string, any>
|
|
351
|
-
): Promise<[Document, number][]> {
|
|
352
|
-
// Generate query embedding
|
|
353
|
-
const embedding = await this.embeddings.embedQuery(query);
|
|
354
|
-
return this.similaritySearchVectorWithScore(embedding, k, filter);
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
/**
|
|
358
|
-
* Delete documents by IDs (blob IDs)
|
|
359
|
-
*/
|
|
360
|
-
async delete(params: { ids: string[] }): Promise<void> {
|
|
361
|
-
for (const blobId of params.ids) {
|
|
362
|
-
try {
|
|
363
|
-
// Remove from memory index
|
|
364
|
-
// Note: This doesn't delete from Walrus or blockchain
|
|
365
|
-
// Those are immutable - we just remove from local index
|
|
366
|
-
await this.memoryIndexService.removeMemory(this.config.userAddress, blobId);
|
|
367
|
-
this.documentMapping.delete(blobId);
|
|
368
|
-
} catch (error) {
|
|
369
|
-
console.error(`Failed to delete document ${blobId}:`, error);
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
/**
|
|
375
|
-
* Maximum marginal relevance search
|
|
376
|
-
* Finds documents that are similar to the query but diverse from each other
|
|
377
|
-
*/
|
|
378
|
-
async maxMarginalRelevanceSearch(
|
|
379
|
-
query: string,
|
|
380
|
-
options: MaxMarginalRelevanceSearchOptions<this["FilterType"]>,
|
|
381
|
-
_callbacks?: Callbacks
|
|
382
|
-
): Promise<Document[]> {
|
|
383
|
-
const k = options.k ?? 4;
|
|
384
|
-
const fetchK = options.fetchK ?? 20;
|
|
385
|
-
const lambda = options.lambda ?? 0.5;
|
|
386
|
-
|
|
387
|
-
// Generate query embedding
|
|
388
|
-
const queryEmbedding = await this.embeddings.embedQuery(query);
|
|
389
|
-
|
|
390
|
-
// Handle filter (can be string or object from base class)
|
|
391
|
-
const filterObj = typeof options.filter === 'object' && options.filter !== null
|
|
392
|
-
? options.filter as Record<string, any>
|
|
393
|
-
: {};
|
|
394
|
-
|
|
395
|
-
// Fetch more results than needed
|
|
396
|
-
const searchQuery: MemorySearchQuery = {
|
|
397
|
-
vector: queryEmbedding,
|
|
398
|
-
userAddress: this.config.userAddress,
|
|
399
|
-
k: fetchK,
|
|
400
|
-
threshold: filterObj.minSimilarity || 0.0,
|
|
401
|
-
categories: filterObj.categories,
|
|
402
|
-
tags: filterObj.tags,
|
|
403
|
-
includeContent: false,
|
|
404
|
-
diversityFactor: 1 - lambda, // Higher diversity factor = more diverse results
|
|
405
|
-
};
|
|
406
|
-
|
|
407
|
-
const results = await this.memoryIndexService.searchMemories(searchQuery);
|
|
408
|
-
|
|
409
|
-
// Convert to documents and apply MMR
|
|
410
|
-
const documents = results.slice(0, k).map((result: MemorySearchResult) => {
|
|
411
|
-
const pageContent = result.metadata?.topic || result.metadata?.category || '';
|
|
412
|
-
|
|
413
|
-
return new Document({
|
|
414
|
-
pageContent,
|
|
415
|
-
metadata: {
|
|
416
|
-
...result.metadata,
|
|
417
|
-
blobId: result.blobId,
|
|
418
|
-
memoryId: result.memoryId,
|
|
419
|
-
similarity: result.similarity,
|
|
420
|
-
relevanceScore: result.relevanceScore,
|
|
421
|
-
},
|
|
422
|
-
});
|
|
423
|
-
});
|
|
424
|
-
|
|
425
|
-
return documents;
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
/**
|
|
429
|
-
* Get statistics about the vector store
|
|
430
|
-
*/
|
|
431
|
-
async getStats(): Promise<{
|
|
432
|
-
totalMemories: number;
|
|
433
|
-
categoryCounts: Record<string, number>;
|
|
434
|
-
importanceDistribution: Record<number, number>;
|
|
435
|
-
averageImportance: number;
|
|
436
|
-
oldestMemory: Date | null;
|
|
437
|
-
newestMemory: Date | null;
|
|
438
|
-
indexSize: number;
|
|
439
|
-
}> {
|
|
440
|
-
return this.memoryIndexService.getIndexStats(this.config.userAddress);
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
/**
|
|
444
|
-
* Clear all memories for the current user
|
|
445
|
-
* Note: This only clears the local index, not blockchain or Walrus
|
|
446
|
-
*/
|
|
447
|
-
async clear(): Promise<void> {
|
|
448
|
-
await this.memoryIndexService.clearUserIndex(this.config.userAddress);
|
|
449
|
-
this.documentMapping.clear();
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
// Required by VectorStore abstract class
|
|
453
|
-
_vectorstoreType(): string {
|
|
454
|
-
return 'pdw';
|
|
455
|
-
}
|
|
456
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* PDWVectorStore - LangChain VectorStore implementation for Personal Data Wallet
|
|
3
|
+
*
|
|
4
|
+
* Integrates PDW's decentralized storage (Walrus) and vector search (HNSW) with LangChain.
|
|
5
|
+
* Enables RAG workflows with memories stored on Sui blockchain and Walrus.
|
|
6
|
+
*
|
|
7
|
+
* Features:
|
|
8
|
+
* - Decentralized vector storage on Walrus
|
|
9
|
+
* - HNSW-powered similarity search (browser-compatible WebAssembly)
|
|
10
|
+
* - Blockchain-backed ownership via Sui
|
|
11
|
+
* - SEAL encryption for privacy (optional)
|
|
12
|
+
* - Memory metadata and filtering
|
|
13
|
+
*
|
|
14
|
+
* @example
|
|
15
|
+
* ```typescript
|
|
16
|
+
* import { PDWVectorStore, PDWEmbeddings } from 'personal-data-wallet-sdk/langchain';
|
|
17
|
+
* import { useCurrentAccount, useSuiClient } from '@mysten/dapp-kit';
|
|
18
|
+
*
|
|
19
|
+
* const embeddings = new PDWEmbeddings({ geminiApiKey });
|
|
20
|
+
* const vectorStore = new PDWVectorStore(embeddings, {
|
|
21
|
+
* userAddress: account.address,
|
|
22
|
+
* packageId: '0x...',
|
|
23
|
+
* walrusAggregator: 'https://aggregator.walrus-testnet.walrus.space',
|
|
24
|
+
* });
|
|
25
|
+
*
|
|
26
|
+
* // Add documents (requires Sui wallet signing)
|
|
27
|
+
* await vectorStore.addDocuments([
|
|
28
|
+
* { pageContent: 'Hello world', metadata: { category: 'greeting' } }
|
|
29
|
+
* ], { account, signAndExecute, client });
|
|
30
|
+
*
|
|
31
|
+
* // Search
|
|
32
|
+
* const results = await vectorStore.similaritySearch('hi', 5);
|
|
33
|
+
* ```
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
import { VectorStore } from '@langchain/core/vectorstores';
|
|
37
|
+
import { Document } from '@langchain/core/documents';
|
|
38
|
+
import type { EmbeddingsInterface } from '@langchain/core/embeddings';
|
|
39
|
+
import type { Callbacks } from '@langchain/core/callbacks/manager';
|
|
40
|
+
import type { MaxMarginalRelevanceSearchOptions } from '@langchain/core/vectorstores';
|
|
41
|
+
import { ClientMemoryManager } from '../client/ClientMemoryManager';
|
|
42
|
+
import type { ClientMemoryManagerConfig } from '../client/ClientMemoryManager';
|
|
43
|
+
import { MemoryIndexService } from '../services/MemoryIndexService';
|
|
44
|
+
import type { MemorySearchQuery, MemorySearchResult } from '../services/MemoryIndexService';
|
|
45
|
+
import { StorageService } from '../services/StorageService';
|
|
46
|
+
import { EmbeddingService } from '../services/EmbeddingService';
|
|
47
|
+
import type { Transaction } from '@mysten/sui/transactions';
|
|
48
|
+
import type { SuiClient } from '@mysten/sui/client';
|
|
49
|
+
|
|
50
|
+
export interface PDWVectorStoreConfig {
|
|
51
|
+
// Required
|
|
52
|
+
userAddress: string;
|
|
53
|
+
|
|
54
|
+
// Sui Configuration
|
|
55
|
+
packageId: string;
|
|
56
|
+
accessRegistryId?: string;
|
|
57
|
+
network?: 'mainnet' | 'testnet' | 'devnet';
|
|
58
|
+
|
|
59
|
+
// Walrus Configuration
|
|
60
|
+
walrusPublisher?: string;
|
|
61
|
+
walrusAggregator?: string;
|
|
62
|
+
walrusNetwork?: 'testnet' | 'mainnet';
|
|
63
|
+
|
|
64
|
+
// AI Configuration (for MemoryIndexService)
|
|
65
|
+
geminiApiKey: string;
|
|
66
|
+
embeddingModel?: string;
|
|
67
|
+
embeddingDimensions?: number;
|
|
68
|
+
|
|
69
|
+
// Vector Index Configuration
|
|
70
|
+
maxElements?: number;
|
|
71
|
+
efConstruction?: number;
|
|
72
|
+
m?: number;
|
|
73
|
+
|
|
74
|
+
// Memory Configuration
|
|
75
|
+
defaultCategory?: string;
|
|
76
|
+
defaultImportance?: number;
|
|
77
|
+
encryptionEnabled?: boolean;
|
|
78
|
+
|
|
79
|
+
// SEAL Configuration
|
|
80
|
+
sealServerObjectIds?: string[];
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export interface PDWAddDocumentOptions {
|
|
84
|
+
// Sui wallet signing (required for adding documents)
|
|
85
|
+
account: { address: string };
|
|
86
|
+
signAndExecute: (params: { transaction: Transaction }, callbacks: {
|
|
87
|
+
onSuccess: (result: any) => void;
|
|
88
|
+
onError: (error: Error) => void;
|
|
89
|
+
}) => void;
|
|
90
|
+
client: SuiClient;
|
|
91
|
+
|
|
92
|
+
// Optional memory metadata
|
|
93
|
+
category?: string;
|
|
94
|
+
importance?: number;
|
|
95
|
+
tags?: string[];
|
|
96
|
+
encrypt?: boolean;
|
|
97
|
+
onProgress?: (status: string) => void;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* LangChain VectorStore implementation using Personal Data Wallet
|
|
102
|
+
*
|
|
103
|
+
* This class wraps PDW's ClientMemoryManager and MemoryIndexService to provide
|
|
104
|
+
* a standard LangChain VectorStore interface while preserving all PDW features:
|
|
105
|
+
* - Decentralized storage on Walrus
|
|
106
|
+
* - Blockchain ownership via Sui
|
|
107
|
+
* - SEAL encryption
|
|
108
|
+
* - HNSW vector search
|
|
109
|
+
*/
|
|
110
|
+
export class PDWVectorStore extends VectorStore {
|
|
111
|
+
private clientMemoryManager!: ClientMemoryManager;
|
|
112
|
+
private memoryIndexService!: MemoryIndexService;
|
|
113
|
+
private storageService!: StorageService;
|
|
114
|
+
private embeddingService!: EmbeddingService;
|
|
115
|
+
private config: PDWVectorStoreConfig;
|
|
116
|
+
|
|
117
|
+
// Track document to blobId mapping
|
|
118
|
+
private documentMapping = new Map<string, string>(); // blobId -> documentId
|
|
119
|
+
|
|
120
|
+
constructor(
|
|
121
|
+
embeddings: EmbeddingsInterface,
|
|
122
|
+
config: PDWVectorStoreConfig
|
|
123
|
+
) {
|
|
124
|
+
super(embeddings, {});
|
|
125
|
+
this.config = config;
|
|
126
|
+
|
|
127
|
+
// Initialize PDW services
|
|
128
|
+
this.initializeServices();
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
private initializeServices() {
|
|
132
|
+
// Initialize ClientMemoryManager
|
|
133
|
+
const managerConfig: ClientMemoryManagerConfig = {
|
|
134
|
+
packageId: this.config.packageId,
|
|
135
|
+
accessRegistryId: this.config.accessRegistryId || '',
|
|
136
|
+
walrusAggregator: this.config.walrusAggregator || 'https://aggregator.walrus-testnet.walrus.space',
|
|
137
|
+
geminiApiKey: this.config.geminiApiKey,
|
|
138
|
+
sealServerObjectIds: this.config.sealServerObjectIds,
|
|
139
|
+
walrusNetwork: this.config.walrusNetwork || 'testnet',
|
|
140
|
+
enableLocalIndexing: true,
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
this.clientMemoryManager = new ClientMemoryManager(managerConfig);
|
|
144
|
+
|
|
145
|
+
// Initialize services
|
|
146
|
+
this.embeddingService = new EmbeddingService({
|
|
147
|
+
apiKey: this.config.geminiApiKey,
|
|
148
|
+
model: this.config.embeddingModel || 'text-embedding-004',
|
|
149
|
+
dimensions: this.config.embeddingDimensions || 3072,
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
this.storageService = new StorageService({
|
|
153
|
+
walrusPublisherUrl: this.config.walrusPublisher,
|
|
154
|
+
walrusAggregatorUrl: this.config.walrusAggregator || 'https://aggregator.walrus-testnet.walrus.space',
|
|
155
|
+
network: (this.config.network === 'devnet' ? 'testnet' : this.config.network) || 'testnet',
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
this.memoryIndexService = new MemoryIndexService(
|
|
159
|
+
this.storageService,
|
|
160
|
+
{
|
|
161
|
+
maxElements: this.config.maxElements || 10000,
|
|
162
|
+
dimension: this.config.embeddingDimensions || 3072,
|
|
163
|
+
efConstruction: this.config.efConstruction || 200,
|
|
164
|
+
m: this.config.m || 16,
|
|
165
|
+
}
|
|
166
|
+
);
|
|
167
|
+
|
|
168
|
+
// Connect embedding service to index service
|
|
169
|
+
this.memoryIndexService.initialize(this.embeddingService, this.storageService);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Create PDWVectorStore from documents (LangChain factory pattern)
|
|
174
|
+
*
|
|
175
|
+
* @param docs - Documents to add
|
|
176
|
+
* @param embeddings - Embeddings interface
|
|
177
|
+
* @param dbConfig - Combined config object containing both PDWVectorStoreConfig and PDWAddDocumentOptions
|
|
178
|
+
*/
|
|
179
|
+
static async fromDocuments(
|
|
180
|
+
docs: Document[],
|
|
181
|
+
embeddings: EmbeddingsInterface,
|
|
182
|
+
dbConfig: PDWVectorStoreConfig & { addOptions?: PDWAddDocumentOptions }
|
|
183
|
+
): Promise<PDWVectorStore> {
|
|
184
|
+
const { addOptions, ...config } = dbConfig;
|
|
185
|
+
const instance = new PDWVectorStore(embeddings, config as PDWVectorStoreConfig);
|
|
186
|
+
|
|
187
|
+
if (addOptions) {
|
|
188
|
+
await instance.addDocuments(docs, addOptions);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
return instance;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Create PDWVectorStore from texts (LangChain factory pattern)
|
|
196
|
+
*
|
|
197
|
+
* @param texts - Text strings to add
|
|
198
|
+
* @param metadatas - Metadata for each text
|
|
199
|
+
* @param embeddings - Embeddings interface
|
|
200
|
+
* @param dbConfig - Combined config object containing both PDWVectorStoreConfig and PDWAddDocumentOptions
|
|
201
|
+
*/
|
|
202
|
+
static async fromTexts(
|
|
203
|
+
texts: string[],
|
|
204
|
+
metadatas: object[] | object,
|
|
205
|
+
embeddings: EmbeddingsInterface,
|
|
206
|
+
dbConfig: PDWVectorStoreConfig & { addOptions?: PDWAddDocumentOptions }
|
|
207
|
+
): Promise<PDWVectorStore> {
|
|
208
|
+
const docs = texts.map((text, i) => {
|
|
209
|
+
const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
|
|
210
|
+
return new Document({ pageContent: text, metadata });
|
|
211
|
+
});
|
|
212
|
+
return PDWVectorStore.fromDocuments(docs, embeddings, dbConfig);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Add documents to the vector store
|
|
217
|
+
*
|
|
218
|
+
* This creates memories on Sui blockchain and stores them on Walrus.
|
|
219
|
+
* Requires wallet signing via PDWAddDocumentOptions.
|
|
220
|
+
*/
|
|
221
|
+
async addDocuments(
|
|
222
|
+
documents: Document[],
|
|
223
|
+
options?: PDWAddDocumentOptions
|
|
224
|
+
): Promise<string[]> {
|
|
225
|
+
if (!options) {
|
|
226
|
+
throw new Error(
|
|
227
|
+
'PDWVectorStore.addDocuments requires options with wallet signing. ' +
|
|
228
|
+
'Pass { account, signAndExecute, client } to sign transactions.'
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const blobIds: string[] = [];
|
|
233
|
+
|
|
234
|
+
for (const doc of documents) {
|
|
235
|
+
try {
|
|
236
|
+
if (options.onProgress) {
|
|
237
|
+
options.onProgress(`Creating memory: ${doc.pageContent.substring(0, 50)}...`);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Create memory using ClientMemoryManager
|
|
241
|
+
// This handles: embedding → encryption → Walrus upload → on-chain registration
|
|
242
|
+
const blobId = await this.clientMemoryManager.createMemory({
|
|
243
|
+
content: doc.pageContent,
|
|
244
|
+
category: options.category || doc.metadata?.category || this.config.defaultCategory || 'general',
|
|
245
|
+
account: options.account,
|
|
246
|
+
signAndExecute: options.signAndExecute,
|
|
247
|
+
client: options.client,
|
|
248
|
+
onProgress: options.onProgress,
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
blobIds.push(blobId);
|
|
252
|
+
this.documentMapping.set(blobId, doc.id || blobId);
|
|
253
|
+
|
|
254
|
+
if (options.onProgress) {
|
|
255
|
+
options.onProgress(`✅ Memory created: ${blobId}`);
|
|
256
|
+
}
|
|
257
|
+
} catch (error) {
|
|
258
|
+
console.error(`Failed to add document:`, error);
|
|
259
|
+
throw new Error(
|
|
260
|
+
`Failed to add document: ${error instanceof Error ? error.message : String(error)}`
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return blobIds;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Add vectors directly to the store
|
|
270
|
+
*
|
|
271
|
+
* This is useful when you already have embeddings and want to avoid regenerating them.
|
|
272
|
+
*/
|
|
273
|
+
async addVectors(
|
|
274
|
+
vectors: number[][],
|
|
275
|
+
documents: Document[],
|
|
276
|
+
options?: PDWAddDocumentOptions
|
|
277
|
+
): Promise<string[]> {
|
|
278
|
+
if (vectors.length !== documents.length) {
|
|
279
|
+
throw new Error('Vectors and documents must have the same length');
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// Note: Currently ClientMemoryManager generates embeddings internally
|
|
283
|
+
// For now, we'll call addDocuments which will regenerate embeddings
|
|
284
|
+
// TODO: Add support for passing pre-computed embeddings to ClientMemoryManager
|
|
285
|
+
return this.addDocuments(documents, options);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Similarity search with score (core LangChain method)
|
|
290
|
+
*/
|
|
291
|
+
async similaritySearchVectorWithScore(
|
|
292
|
+
query: number[],
|
|
293
|
+
k: number,
|
|
294
|
+
filter?: Record<string, any>
|
|
295
|
+
): Promise<[Document, number][]> {
|
|
296
|
+
// Build search query for MemoryIndexService
|
|
297
|
+
const searchQuery: MemorySearchQuery = {
|
|
298
|
+
vector: query,
|
|
299
|
+
userAddress: this.config.userAddress,
|
|
300
|
+
k: k || 5,
|
|
301
|
+
threshold: filter?.minSimilarity || 0.0,
|
|
302
|
+
categories: filter?.category ? [filter.category] : filter?.categories,
|
|
303
|
+
tags: filter?.tags,
|
|
304
|
+
dateRange: filter?.dateRange,
|
|
305
|
+
importanceRange: filter?.importanceRange,
|
|
306
|
+
includeContent: false, // Don't fetch content by default (can be slow with decryption)
|
|
307
|
+
};
|
|
308
|
+
|
|
309
|
+
// Search using MemoryIndexService
|
|
310
|
+
const results = await this.memoryIndexService.searchMemories(searchQuery);
|
|
311
|
+
|
|
312
|
+
// Convert to LangChain format
|
|
313
|
+
return results.map((result: MemorySearchResult) => {
|
|
314
|
+
// Use metadata topic as pageContent if content not included
|
|
315
|
+
const pageContent = result.metadata?.topic || result.metadata?.category || '';
|
|
316
|
+
|
|
317
|
+
const doc = new Document({
|
|
318
|
+
pageContent,
|
|
319
|
+
metadata: {
|
|
320
|
+
...result.metadata,
|
|
321
|
+
blobId: result.blobId,
|
|
322
|
+
memoryId: result.memoryId,
|
|
323
|
+
similarity: result.similarity,
|
|
324
|
+
relevanceScore: result.relevanceScore,
|
|
325
|
+
},
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
return [doc, result.similarity] as [Document, number];
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Similarity search (standard LangChain method)
|
|
334
|
+
*/
|
|
335
|
+
async similaritySearch(
|
|
336
|
+
query: string,
|
|
337
|
+
k: number = 5,
|
|
338
|
+
filter?: Record<string, any>
|
|
339
|
+
): Promise<Document[]> {
|
|
340
|
+
const resultsWithScore = await this.similaritySearchWithScore(query, k, filter);
|
|
341
|
+
return resultsWithScore.map(([doc]) => doc);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* Similarity search with score using query string
|
|
346
|
+
*/
|
|
347
|
+
async similaritySearchWithScore(
|
|
348
|
+
query: string,
|
|
349
|
+
k: number = 5,
|
|
350
|
+
filter?: Record<string, any>
|
|
351
|
+
): Promise<[Document, number][]> {
|
|
352
|
+
// Generate query embedding
|
|
353
|
+
const embedding = await this.embeddings.embedQuery(query);
|
|
354
|
+
return this.similaritySearchVectorWithScore(embedding, k, filter);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Delete documents by IDs (blob IDs)
|
|
359
|
+
*/
|
|
360
|
+
async delete(params: { ids: string[] }): Promise<void> {
|
|
361
|
+
for (const blobId of params.ids) {
|
|
362
|
+
try {
|
|
363
|
+
// Remove from memory index
|
|
364
|
+
// Note: This doesn't delete from Walrus or blockchain
|
|
365
|
+
// Those are immutable - we just remove from local index
|
|
366
|
+
await this.memoryIndexService.removeMemory(this.config.userAddress, blobId);
|
|
367
|
+
this.documentMapping.delete(blobId);
|
|
368
|
+
} catch (error) {
|
|
369
|
+
console.error(`Failed to delete document ${blobId}:`, error);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* Maximum marginal relevance search
|
|
376
|
+
* Finds documents that are similar to the query but diverse from each other
|
|
377
|
+
*/
|
|
378
|
+
async maxMarginalRelevanceSearch(
|
|
379
|
+
query: string,
|
|
380
|
+
options: MaxMarginalRelevanceSearchOptions<this["FilterType"]>,
|
|
381
|
+
_callbacks?: Callbacks
|
|
382
|
+
): Promise<Document[]> {
|
|
383
|
+
const k = options.k ?? 4;
|
|
384
|
+
const fetchK = options.fetchK ?? 20;
|
|
385
|
+
const lambda = options.lambda ?? 0.5;
|
|
386
|
+
|
|
387
|
+
// Generate query embedding
|
|
388
|
+
const queryEmbedding = await this.embeddings.embedQuery(query);
|
|
389
|
+
|
|
390
|
+
// Handle filter (can be string or object from base class)
|
|
391
|
+
const filterObj = typeof options.filter === 'object' && options.filter !== null
|
|
392
|
+
? options.filter as Record<string, any>
|
|
393
|
+
: {};
|
|
394
|
+
|
|
395
|
+
// Fetch more results than needed
|
|
396
|
+
const searchQuery: MemorySearchQuery = {
|
|
397
|
+
vector: queryEmbedding,
|
|
398
|
+
userAddress: this.config.userAddress,
|
|
399
|
+
k: fetchK,
|
|
400
|
+
threshold: filterObj.minSimilarity || 0.0,
|
|
401
|
+
categories: filterObj.categories,
|
|
402
|
+
tags: filterObj.tags,
|
|
403
|
+
includeContent: false,
|
|
404
|
+
diversityFactor: 1 - lambda, // Higher diversity factor = more diverse results
|
|
405
|
+
};
|
|
406
|
+
|
|
407
|
+
const results = await this.memoryIndexService.searchMemories(searchQuery);
|
|
408
|
+
|
|
409
|
+
// Convert to documents and apply MMR
|
|
410
|
+
const documents = results.slice(0, k).map((result: MemorySearchResult) => {
|
|
411
|
+
const pageContent = result.metadata?.topic || result.metadata?.category || '';
|
|
412
|
+
|
|
413
|
+
return new Document({
|
|
414
|
+
pageContent,
|
|
415
|
+
metadata: {
|
|
416
|
+
...result.metadata,
|
|
417
|
+
blobId: result.blobId,
|
|
418
|
+
memoryId: result.memoryId,
|
|
419
|
+
similarity: result.similarity,
|
|
420
|
+
relevanceScore: result.relevanceScore,
|
|
421
|
+
},
|
|
422
|
+
});
|
|
423
|
+
});
|
|
424
|
+
|
|
425
|
+
return documents;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* Get statistics about the vector store
|
|
430
|
+
*/
|
|
431
|
+
async getStats(): Promise<{
|
|
432
|
+
totalMemories: number;
|
|
433
|
+
categoryCounts: Record<string, number>;
|
|
434
|
+
importanceDistribution: Record<number, number>;
|
|
435
|
+
averageImportance: number;
|
|
436
|
+
oldestMemory: Date | null;
|
|
437
|
+
newestMemory: Date | null;
|
|
438
|
+
indexSize: number;
|
|
439
|
+
}> {
|
|
440
|
+
return this.memoryIndexService.getIndexStats(this.config.userAddress);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* Clear all memories for the current user
|
|
445
|
+
* Note: This only clears the local index, not blockchain or Walrus
|
|
446
|
+
*/
|
|
447
|
+
async clear(): Promise<void> {
|
|
448
|
+
await this.memoryIndexService.clearUserIndex(this.config.userAddress);
|
|
449
|
+
this.documentMapping.clear();
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
// Required by VectorStore abstract class
|
|
453
|
+
_vectorstoreType(): string {
|
|
454
|
+
return 'pdw';
|
|
455
|
+
}
|
|
456
|
+
}
|