@cmdoss/memwal-sdk 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +547 -547
- package/BENCHMARKS.md +238 -238
- package/README.md +181 -181
- package/dist/ai-sdk/tools.d.ts +2 -2
- package/dist/ai-sdk/tools.js +2 -2
- package/dist/client/PersonalDataWallet.d.ts.map +1 -1
- package/dist/client/SimplePDWClient.d.ts +1 -1
- package/dist/client/SimplePDWClient.d.ts.map +1 -1
- package/dist/client/SimplePDWClient.js +16 -7
- package/dist/client/SimplePDWClient.js.map +1 -1
- package/dist/client/namespaces/EmbeddingsNamespace.d.ts +1 -1
- package/dist/client/namespaces/EmbeddingsNamespace.js +1 -1
- package/dist/client/namespaces/MemoryNamespace.d.ts +27 -0
- package/dist/client/namespaces/MemoryNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/MemoryNamespace.js +104 -0
- package/dist/client/namespaces/MemoryNamespace.js.map +1 -1
- package/dist/client/namespaces/consolidated/AINamespace.d.ts +2 -2
- package/dist/client/namespaces/consolidated/AINamespace.js +2 -2
- package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/consolidated/BlockchainNamespace.js +22 -2
- package/dist/client/namespaces/consolidated/BlockchainNamespace.js.map +1 -1
- package/dist/graph/GraphService.js +1 -1
- package/dist/graph/GraphService.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/retrieval/MemoryRetrievalService.d.ts +31 -0
- package/dist/retrieval/MemoryRetrievalService.d.ts.map +1 -1
- package/dist/retrieval/MemoryRetrievalService.js +44 -4
- package/dist/retrieval/MemoryRetrievalService.js.map +1 -1
- package/dist/services/EmbeddingService.d.ts +28 -1
- package/dist/services/EmbeddingService.d.ts.map +1 -1
- package/dist/services/EmbeddingService.js +54 -0
- package/dist/services/EmbeddingService.js.map +1 -1
- package/dist/services/IndexManager.d.ts +5 -1
- package/dist/services/IndexManager.d.ts.map +1 -1
- package/dist/services/IndexManager.js +17 -40
- package/dist/services/IndexManager.js.map +1 -1
- package/dist/services/QueryService.js +1 -1
- package/dist/services/QueryService.js.map +1 -1
- package/dist/services/StorageService.d.ts +10 -0
- package/dist/services/StorageService.d.ts.map +1 -1
- package/dist/services/StorageService.js +13 -0
- package/dist/services/StorageService.js.map +1 -1
- package/dist/services/storage/QuiltBatchManager.d.ts +101 -1
- package/dist/services/storage/QuiltBatchManager.d.ts.map +1 -1
- package/dist/services/storage/QuiltBatchManager.js +410 -20
- package/dist/services/storage/QuiltBatchManager.js.map +1 -1
- package/dist/services/storage/index.d.ts +1 -1
- package/dist/services/storage/index.d.ts.map +1 -1
- package/dist/services/storage/index.js.map +1 -1
- package/dist/utils/LRUCache.d.ts +106 -0
- package/dist/utils/LRUCache.d.ts.map +1 -0
- package/dist/utils/LRUCache.js +281 -0
- package/dist/utils/LRUCache.js.map +1 -0
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +2 -0
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/memoryIndexOnChain.d.ts +212 -0
- package/dist/utils/memoryIndexOnChain.d.ts.map +1 -0
- package/dist/utils/memoryIndexOnChain.js +312 -0
- package/dist/utils/memoryIndexOnChain.js.map +1 -0
- package/dist/utils/rebuildIndexNode.d.ts +29 -0
- package/dist/utils/rebuildIndexNode.d.ts.map +1 -1
- package/dist/utils/rebuildIndexNode.js +366 -98
- package/dist/utils/rebuildIndexNode.js.map +1 -1
- package/dist/vector/HnswWasmService.d.ts +20 -5
- package/dist/vector/HnswWasmService.d.ts.map +1 -1
- package/dist/vector/HnswWasmService.js +73 -40
- package/dist/vector/HnswWasmService.js.map +1 -1
- package/dist/vector/IHnswService.d.ts +10 -1
- package/dist/vector/IHnswService.d.ts.map +1 -1
- package/dist/vector/IHnswService.js.map +1 -1
- package/dist/vector/NodeHnswService.d.ts +16 -0
- package/dist/vector/NodeHnswService.d.ts.map +1 -1
- package/dist/vector/NodeHnswService.js +84 -5
- package/dist/vector/NodeHnswService.js.map +1 -1
- package/dist/vector/createHnswService.d.ts +1 -1
- package/dist/vector/createHnswService.js +1 -1
- package/dist/vector/index.d.ts +1 -1
- package/dist/vector/index.js +1 -1
- package/package.json +157 -157
- package/src/access/index.ts +8 -8
- package/src/aggregation/index.ts +8 -8
- package/src/ai-sdk/tools.ts +2 -2
- package/src/client/SimplePDWClient.ts +23 -8
- package/src/client/namespaces/EmbeddingsNamespace.ts +1 -1
- package/src/client/namespaces/MemoryNamespace.ts +137 -0
- package/src/client/namespaces/consolidated/AINamespace.ts +2 -2
- package/src/client/namespaces/consolidated/BlockchainNamespace.ts +20 -2
- package/src/client/signers/DappKitSigner.ts +207 -207
- package/src/core/types/index.ts +1 -1
- package/src/generated/pdw/deps/sui/object.ts +12 -12
- package/src/generated/pdw/deps/sui/vec_map.ts +32 -32
- package/src/generated/pdw/memory.ts +1087 -1087
- package/src/generated/pdw/wallet.ts +123 -123
- package/src/generated/utils/index.ts +159 -159
- package/src/graph/GraphService.ts +1 -1
- package/src/index.ts +25 -1
- package/src/permissions/index.ts +9 -9
- package/src/retrieval/MemoryRetrievalService.ts +78 -4
- package/src/services/EmbeddingService.ts +66 -1
- package/src/services/IndexManager.ts +18 -45
- package/src/services/QueryService.ts +1 -1
- package/src/services/StorageService.ts +15 -0
- package/src/services/storage/QuiltBatchManager.ts +492 -22
- package/src/services/storage/index.ts +6 -1
- package/src/utils/LRUCache.ts +378 -0
- package/src/utils/index.ts +8 -0
- package/src/utils/memoryIndexOnChain.ts +507 -0
- package/src/utils/rebuildIndexNode.ts +453 -106
- package/src/vector/HnswWasmService.ts +95 -43
- package/src/vector/IHnswService.ts +10 -1
- package/src/vector/NodeHnswService.ts +103 -5
- package/src/vector/createHnswService.ts +1 -1
- package/src/vector/index.ts +1 -1
- package/src/wallet/index.ts +17 -17
package/ARCHITECTURE.md
CHANGED
|
@@ -1,547 +1,547 @@
|
|
|
1
|
-
# MemWal SDK Architecture
|
|
2
|
-
|
|
3
|
-
Detailed workflow diagrams and architecture documentation for MemWal (`@cmdoss/memwal-sdk`).
|
|
4
|
-
|
|
5
|
-
## Table of Contents
|
|
6
|
-
|
|
7
|
-
- [Architecture Overview](#architecture-overview)
|
|
8
|
-
- [Memory Creation Flow](#memory-creation-flow)
|
|
9
|
-
- [Search Flow](#search-flow)
|
|
10
|
-
- [Batch Upload (Quilt)](#batch-upload-quilt)
|
|
11
|
-
- [HNSW Vector Indexing](#hnsw-vector-indexing)
|
|
12
|
-
- [SEAL Encryption Flow](#seal-encryption-flow)
|
|
13
|
-
- [API Namespaces](#api-namespaces)
|
|
14
|
-
|
|
15
|
-
---
|
|
16
|
-
|
|
17
|
-
## Architecture Overview
|
|
18
|
-
|
|
19
|
-
```text
|
|
20
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
21
|
-
│ MemWal SDK Architecture │
|
|
22
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
23
|
-
|
|
24
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
25
|
-
│ SimplePDWClient │
|
|
26
|
-
│ ┌─────────────────────────────────────────────────────────────────────────┐│
|
|
27
|
-
│ │ Primary Namespaces ││
|
|
28
|
-
│ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ ││
|
|
29
|
-
│ │ │ memory │ │ search │ │ index │ │ ai │ │ graph │ ││
|
|
30
|
-
│ │ │ │ │ │ │ │ │ │ │ │ ││
|
|
31
|
-
│ │ │• create │ │• vector │ │• add │ │• embed │ │• extract │ ││
|
|
32
|
-
│ │ │• createBatch│• hybrid │ │• search │ │• classify│ │• query │ ││
|
|
33
|
-
│ │ │• get │ │• byCategory│ │• save │ │• shouldSave│ │ ││
|
|
34
|
-
│ │ │• delete │ │• semantic│ │• load │ │ │ │ │ ││
|
|
35
|
-
│ │ └──────────┘ └──────────┘ └──────────┘ └──────────┘ └──────────┘ ││
|
|
36
|
-
│ └─────────────────────────────────────────────────────────────────────────┘│
|
|
37
|
-
│ ┌─────────────────────────────────────────────────────────────────────────┐│
|
|
38
|
-
│ │ Service Container ││
|
|
39
|
-
│ │ embedding | memoryIndex | storage | tx | encryption | capability ││
|
|
40
|
-
│ └─────────────────────────────────────────────────────────────────────────┘│
|
|
41
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
42
|
-
│
|
|
43
|
-
┌───────────────┼───────────────┐
|
|
44
|
-
│ │ │
|
|
45
|
-
▼ ▼ ▼
|
|
46
|
-
┌─────────────────────┐ ┌─────────────────┐ ┌─────────────────────┐
|
|
47
|
-
│ Embedding API │ │ Sui Blockchain │ │ Walrus Storage │
|
|
48
|
-
│ │ │ │ │ │
|
|
49
|
-
│ • OpenRouter │ │ • Testnet │ │ • Publisher API │
|
|
50
|
-
│ • Gemini │ │ • Mainnet │ │ • Aggregator API │
|
|
51
|
-
│ • 3072 dimensions │ │ • Package ID │ │ • Quilt Batching │
|
|
52
|
-
└─────────────────────┘ └─────────────────┘ └─────────────────────┘
|
|
53
|
-
│
|
|
54
|
-
┌───────────────┼───────────────┐
|
|
55
|
-
│ │ │
|
|
56
|
-
▼ ▼ ▼
|
|
57
|
-
┌─────────────────────┐ ┌─────────────────┐ ┌─────────────────────┐
|
|
58
|
-
│ HNSW Vector Index │ │ SEAL Encryption│ │ Knowledge Graph │
|
|
59
|
-
│ │ │ │ │ │
|
|
60
|
-
│ • hnswlib-node │ │ • Threshold IBE │ │ • Entity extraction │
|
|
61
|
-
│ • hnswlib-wasm │ │ • 2-of-N decrypt│ │ • Relationships │
|
|
62
|
-
│ • Auto-detect env │ │ • Mysten SEAL │ │ • Gemini-powered │
|
|
63
|
-
└─────────────────────┘ └─────────────────┘ └─────────────────────┘
|
|
64
|
-
```
|
|
65
|
-
|
|
66
|
-
---
|
|
67
|
-
|
|
68
|
-
## Memory Creation Flow
|
|
69
|
-
|
|
70
|
-
```text
|
|
71
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
72
|
-
│ MemWal Memory Creation Workflow │
|
|
73
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
74
|
-
|
|
75
|
-
User Input: "I am working at CommandOSS as a software engineer"
|
|
76
|
-
│
|
|
77
|
-
▼
|
|
78
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
79
|
-
│ STEP 1: AI Pre-check (Optional) │
|
|
80
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
81
|
-
│ pdw.ai.shouldSave(content) → true/false │
|
|
82
|
-
│ │
|
|
83
|
-
│ • Analyzes content relevance │
|
|
84
|
-
│ • Filters noise/spam │
|
|
85
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
86
|
-
│
|
|
87
|
-
▼
|
|
88
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
89
|
-
│ STEP 2: AI Classification │
|
|
90
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
91
|
-
│ pdw.ai.classify(content) → { category, importance, topic, summary } │
|
|
92
|
-
│ │
|
|
93
|
-
│ Output: { category: "fact", importance: 8, topic: "employment" } │
|
|
94
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
95
|
-
│
|
|
96
|
-
▼
|
|
97
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
98
|
-
│ STEP 3: Embedding Generation │
|
|
99
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
100
|
-
│ pdw.ai.embed(content) → Float32Array[3072] │
|
|
101
|
-
│ │
|
|
102
|
-
│ • OpenRouter or Gemini API │
|
|
103
|
-
│ • 3072 dimensions (text-embedding-3-large) │
|
|
104
|
-
│ • Normalized vector for cosine similarity │
|
|
105
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
106
|
-
│
|
|
107
|
-
▼
|
|
108
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
109
|
-
│ STEP 4: Walrus Upload │
|
|
110
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
111
|
-
│ Memory Package uploaded to Walrus: │
|
|
112
|
-
│ { │
|
|
113
|
-
│ content: "I am working at CommandOSS...", │
|
|
114
|
-
│ embedding: [0.0234, -0.0156, ...], // 3072 dims │
|
|
115
|
-
│ metadata: { category, importance, topic }, │
|
|
116
|
-
│ timestamp: 1701705600000, │
|
|
117
|
-
│ identity: "0xb59f00b2454bef14d538b..." │
|
|
118
|
-
│ } │
|
|
119
|
-
│ │
|
|
120
|
-
│ Output: blobId = "KEG_kj_5nx8wr3eJFZwIkJtjKvEGaUNMvTvhDYddaPU" │
|
|
121
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
122
|
-
│
|
|
123
|
-
▼
|
|
124
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
125
|
-
│ STEP 5: On-chain Registration (Sui Blockchain) │
|
|
126
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
127
|
-
│ Transaction: memory::create_memory_record │
|
|
128
|
-
│ │
|
|
129
|
-
│ Parameters: │
|
|
130
|
-
│ • blob_id: "KEG_kj_5nx8wr3eJFZwIkJtjKvEGaUNMvTvhDYddaPU" │
|
|
131
|
-
│ • category: "fact" │
|
|
132
|
-
│ • importance: 8 │
|
|
133
|
-
│ • content_hash: keccak256(content) │
|
|
134
|
-
│ │
|
|
135
|
-
│ Output: MemoryRecord { id: "0x6bce7d4140b7e6572...", owner: "0xb59f..." } │
|
|
136
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
137
|
-
│
|
|
138
|
-
▼
|
|
139
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
140
|
-
│ STEP 6: Local HNSW Indexing │
|
|
141
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
142
|
-
│ Auto-detected implementation: │
|
|
143
|
-
│ • Node.js: hnswlib-node (native C++, fastest) │
|
|
144
|
-
│ • Browser: hnswlib-wasm (WebAssembly fallback) │
|
|
145
|
-
│ │
|
|
146
|
-
│ Index entry (NO content stored for privacy): │
|
|
147
|
-
│ { memoryId, blobId, category, importance, timestamp, vectorId } │
|
|
148
|
-
│ │
|
|
149
|
-
│ Persistence: │
|
|
150
|
-
│ • Node.js: Filesystem (.pdw-indexes/) │
|
|
151
|
-
│ • Browser: IndexedDB │
|
|
152
|
-
│ • Cloud: Optional Walrus backup │
|
|
153
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
154
|
-
```
|
|
155
|
-
|
|
156
|
-
### Usage
|
|
157
|
-
|
|
158
|
-
```typescript
|
|
159
|
-
import { SimplePDWClient } from '@cmdoss/memwal-sdk';
|
|
160
|
-
|
|
161
|
-
const pdw = new SimplePDWClient({
|
|
162
|
-
signer: keypair,
|
|
163
|
-
network: 'testnet',
|
|
164
|
-
packageId: '0x...',
|
|
165
|
-
embedding: { provider: 'openrouter', apiKey: '...' }
|
|
166
|
-
});
|
|
167
|
-
|
|
168
|
-
await pdw.ready();
|
|
169
|
-
|
|
170
|
-
// Create single memory
|
|
171
|
-
const memory = await pdw.memory.create('I work at CommandOSS', {
|
|
172
|
-
category: 'fact',
|
|
173
|
-
importance: 8
|
|
174
|
-
});
|
|
175
|
-
|
|
176
|
-
// Create batch (uses Quilt for ~90% gas savings)
|
|
177
|
-
const memories = await pdw.memory.createBatch([
|
|
178
|
-
'Memory 1 content',
|
|
179
|
-
'Memory 2 content',
|
|
180
|
-
'Memory 3 content'
|
|
181
|
-
]);
|
|
182
|
-
```
|
|
183
|
-
|
|
184
|
-
---
|
|
185
|
-
|
|
186
|
-
## Search Flow
|
|
187
|
-
|
|
188
|
-
```text
|
|
189
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
190
|
-
│ MemWal Search Workflow │
|
|
191
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
192
|
-
|
|
193
|
-
Query: "What company do I work for?"
|
|
194
|
-
│
|
|
195
|
-
▼
|
|
196
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
197
|
-
│ STEP 1: Query Embedding │
|
|
198
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
199
|
-
│ pdw.ai.embed("What company do I work for?") │
|
|
200
|
-
│ Output: Float32Array[3072] │
|
|
201
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
202
|
-
│
|
|
203
|
-
▼
|
|
204
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
205
|
-
│ STEP 2: HNSW Search (Local Index) │
|
|
206
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
207
|
-
│ Algorithm: Hierarchical Navigable Small World │
|
|
208
|
-
│ Complexity: O(log n) │
|
|
209
|
-
│ │
|
|
210
|
-
│ Returns top-k nearest neighbors by cosine similarity: │
|
|
211
|
-
│ ┌────────┬────────────────────────────────────────────┬───────────┐ │
|
|
212
|
-
│ │ Rank │ Memory ID │ Score │ │
|
|
213
|
-
│ ├────────┼────────────────────────────────────────────┼───────────┤ │
|
|
214
|
-
│ │ 1 │ 0x6bce7d4140b7e6572df79aa1c8b12abfb... │ 0.8542 │ │
|
|
215
|
-
│ │ 2 │ 0xac333f3c2de375da1fbbee1bdada24ff... │ 0.7231 │ │
|
|
216
|
-
│ │ 3 │ 0xbb7a63c4a7a62d2a0b083bc02ee96593... │ 0.6890 │ │
|
|
217
|
-
│ └────────┴────────────────────────────────────────────┴───────────┘ │
|
|
218
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
219
|
-
│
|
|
220
|
-
▼
|
|
221
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
222
|
-
│ STEP 3: Content Retrieval (Walrus) │
|
|
223
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
224
|
-
│ For each result, fetch content from Walrus: │
|
|
225
|
-
│ │
|
|
226
|
-
│ • Download MemoryPackage from blobId │
|
|
227
|
-
│ • Decrypt if encrypted (SEAL) │
|
|
228
|
-
│ • Cache locally for subsequent access │
|
|
229
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
230
|
-
│
|
|
231
|
-
▼
|
|
232
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
233
|
-
│ RESULT │
|
|
234
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
235
|
-
│ [ │
|
|
236
|
-
│ { │
|
|
237
|
-
│ id: "0x6bce7d4140b7e6572df79aa1c8b12abfb...", │
|
|
238
|
-
│ content: "I am working at CommandOSS as a software engineer", │
|
|
239
|
-
│ score: 0.8542, │
|
|
240
|
-
│ category: "fact", │
|
|
241
|
-
│ importance: 8 │
|
|
242
|
-
│ }, │
|
|
243
|
-
│ ... │
|
|
244
|
-
│ ] │
|
|
245
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
246
|
-
```
|
|
247
|
-
|
|
248
|
-
### Search Methods
|
|
249
|
-
|
|
250
|
-
```typescript
|
|
251
|
-
// Vector search (semantic)
|
|
252
|
-
const results = await pdw.search.vector('work experience', { limit: 5 });
|
|
253
|
-
|
|
254
|
-
// Filter by category
|
|
255
|
-
const facts = await pdw.search.byCategory('fact', { limit: 10 });
|
|
256
|
-
|
|
257
|
-
// Hybrid search (vector + category filter)
|
|
258
|
-
const filtered = await pdw.search.hybrid('work', { category: 'fact' });
|
|
259
|
-
|
|
260
|
-
// With content fetched
|
|
261
|
-
const withContent = await pdw.search.withContent(results);
|
|
262
|
-
```
|
|
263
|
-
|
|
264
|
-
---
|
|
265
|
-
|
|
266
|
-
## Batch Upload (Quilt)
|
|
267
|
-
|
|
268
|
-
Walrus Quilt enables batch uploads with ~90% gas savings.
|
|
269
|
-
|
|
270
|
-
```text
|
|
271
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
272
|
-
│ Quilt Batch Upload Flow │
|
|
273
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
274
|
-
|
|
275
|
-
Individual Upload: Quilt Batch Upload:
|
|
276
|
-
┌─────────────┐ ┌─────────────┐
|
|
277
|
-
│ Memory 1 │──► Transaction 1 │ Memory 1 │
|
|
278
|
-
└─────────────┘ │ Memory 2 │──► Single Transaction
|
|
279
|
-
┌─────────────┐ │ Memory 3 │ (Quilt)
|
|
280
|
-
│ Memory 2 │──► Transaction 2 │ Memory 4 │
|
|
281
|
-
└─────────────┘ │ Memory 5 │
|
|
282
|
-
┌─────────────┐ └─────────────┘
|
|
283
|
-
│ Memory 3 │──► Transaction 3
|
|
284
|
-
└─────────────┘ Gas: 1x base fee
|
|
285
|
-
vs
|
|
286
|
-
Gas: 3x base fee Individual: 5x base fee
|
|
287
|
-
Savings: ~90%
|
|
288
|
-
|
|
289
|
-
Quilt Structure:
|
|
290
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
291
|
-
│ quiltId: "abc123..." │
|
|
292
|
-
│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │
|
|
293
|
-
│ │ quiltPatchId: 1 │ │ quiltPatchId: 2 │ │ quiltPatchId: 3 │ │
|
|
294
|
-
│ │ identifier: m1 │ │ identifier: m2 │ │ identifier: m3 │ │
|
|
295
|
-
│ │ tags: {...} │ │ tags: {...} │ │ tags: {...} │ │
|
|
296
|
-
│ │ content: ... │ │ content: ... │ │ content: ... │ │
|
|
297
|
-
│ └─────────────────┘ └─────────────────┘ └─────────────────┘ │
|
|
298
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
299
|
-
```
|
|
300
|
-
|
|
301
|
-
### Usage
|
|
302
|
-
|
|
303
|
-
```typescript
|
|
304
|
-
// Batch create memories (automatically uses Quilt)
|
|
305
|
-
const results = await pdw.memory.createBatch([
|
|
306
|
-
'First memory content',
|
|
307
|
-
'Second memory content',
|
|
308
|
-
'Third memory content'
|
|
309
|
-
]);
|
|
310
|
-
|
|
311
|
-
// Results include quilt information
|
|
312
|
-
results.forEach(r => {
|
|
313
|
-
console.log(`ID: ${r.id}, BlobId: ${r.blobId}`);
|
|
314
|
-
});
|
|
315
|
-
```
|
|
316
|
-
|
|
317
|
-
---
|
|
318
|
-
|
|
319
|
-
## HNSW Vector Indexing
|
|
320
|
-
|
|
321
|
-
The SDK auto-detects the environment and uses the optimal HNSW implementation.
|
|
322
|
-
|
|
323
|
-
```text
|
|
324
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
325
|
-
│ Environment-Aware HNSW Factory │
|
|
326
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
327
|
-
|
|
328
|
-
createHnswService()
|
|
329
|
-
│
|
|
330
|
-
┌─────────────┴─────────────┐
|
|
331
|
-
│ │
|
|
332
|
-
isBrowser()? isNode()?
|
|
333
|
-
│ │
|
|
334
|
-
▼ ▼
|
|
335
|
-
┌─────────────────────────┐ ┌─────────────────────────┐
|
|
336
|
-
│ BrowserHnswIndexService│ │ NodeHnswService │
|
|
337
|
-
│ (hnswlib-wasm) │ │ (hnswlib-node) │
|
|
338
|
-
│ │ │ │
|
|
339
|
-
│ • WebAssembly │ │ • Native C++ bindings │
|
|
340
|
-
│ • IndexedDB persistence│ │ • Filesystem persistence│
|
|
341
|
-
│ • ~50% slower than node│ │ • Fastest performance │
|
|
342
|
-
│ • Universal fallback │ │ • Requires build tools │
|
|
343
|
-
└─────────────────────────┘ └─────────────────────────┘
|
|
344
|
-
|
|
345
|
-
Singleton Pattern:
|
|
346
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
347
|
-
│ First call: Creates new instance and initializes │
|
|
348
|
-
│ Subsequent calls: Returns existing singleton │
|
|
349
|
-
│ │
|
|
350
|
-
│ Benefits: │
|
|
351
|
-
│ • Prevents redundant index loading │
|
|
352
|
-
│ • Shared index across all SDK clients │
|
|
353
|
-
│ • Automatic environment detection │
|
|
354
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
355
|
-
```
|
|
356
|
-
|
|
357
|
-
### Index Configuration
|
|
358
|
-
|
|
359
|
-
```typescript
|
|
360
|
-
// Default configuration
|
|
361
|
-
const indexConfig = {
|
|
362
|
-
dimension: 3072, // Embedding dimensions
|
|
363
|
-
maxElements: 10000, // Maximum vectors
|
|
364
|
-
efConstruction: 200, // Build-time quality
|
|
365
|
-
m: 16, // Connections per layer
|
|
366
|
-
spaceType: 'cosine' // Distance metric
|
|
367
|
-
};
|
|
368
|
-
|
|
369
|
-
// Access via namespace
|
|
370
|
-
const stats = await pdw.index.getStats();
|
|
371
|
-
// { currentCount: 150, dimension: 3072, maxElements: 10000 }
|
|
372
|
-
|
|
373
|
-
// Manual save/load
|
|
374
|
-
await pdw.index.save(); // Save to Walrus for backup
|
|
375
|
-
await pdw.index.load(); // Load from Walrus backup
|
|
376
|
-
```
|
|
377
|
-
|
|
378
|
-
---
|
|
379
|
-
|
|
380
|
-
## SEAL Encryption Flow
|
|
381
|
-
|
|
382
|
-
```text
|
|
383
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
384
|
-
│ SEAL Encryption Workflow │
|
|
385
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
386
|
-
|
|
387
|
-
ENCRYPTION
|
|
388
|
-
│
|
|
389
|
-
▼
|
|
390
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
391
|
-
│ STEP 1: Create MemoryCap (On-chain) │
|
|
392
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
393
|
-
│ Creates on-chain capability object for key derivation │
|
|
394
|
-
│ │
|
|
395
|
-
│ MemoryCap { │
|
|
396
|
-
│ id: "0x05ac7bdc83308ea2cea429afc9fd6bbc91838b...", │
|
|
397
|
-
│ owner: "0xb59f00b2454bef14d538b...", │
|
|
398
|
-
│ nonce: [random 32 bytes] ← Used for key derivation │
|
|
399
|
-
│ } │
|
|
400
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
401
|
-
│
|
|
402
|
-
▼
|
|
403
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
404
|
-
│ STEP 2: SEAL Encrypt │
|
|
405
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
406
|
-
│ • Identity-Based Encryption (IBE) │
|
|
407
|
-
│ • Threshold: 2 of N key servers must agree │
|
|
408
|
-
│ • Key servers: Mysten Labs operated │
|
|
409
|
-
│ │
|
|
410
|
-
│ Output: { encryptedObject, keyId, nonce } │
|
|
411
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
412
|
-
│
|
|
413
|
-
▼
|
|
414
|
-
Upload to Walrus
|
|
415
|
-
│
|
|
416
|
-
DECRYPTION
|
|
417
|
-
│
|
|
418
|
-
▼
|
|
419
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
420
|
-
│ STEP 3: Request Decryption Key │
|
|
421
|
-
├─────────────────────────────────────────────────────────────────────────────┤
|
|
422
|
-
│ 1. Build seal_approve transaction with MemoryCap │
|
|
423
|
-
│ 2. Key servers verify: │
|
|
424
|
-
│ • Caller owns the MemoryCap │
|
|
425
|
-
│ • key_id matches MemoryCap's derived key │
|
|
426
|
-
│ 3. If verified, key servers release decryption shares │
|
|
427
|
-
│ 4. Combine shares (threshold) to decrypt │
|
|
428
|
-
│ │
|
|
429
|
-
│ Output: Decrypted content │
|
|
430
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
431
|
-
```
|
|
432
|
-
|
|
433
|
-
---
|
|
434
|
-
|
|
435
|
-
## API Namespaces
|
|
436
|
-
|
|
437
|
-
### Primary Namespaces
|
|
438
|
-
|
|
439
|
-
| Namespace | Methods | Description |
|
|
440
|
-
|-----------|---------|-------------|
|
|
441
|
-
| `pdw.memory` | create, createBatch, get, delete, list | Memory CRUD operations |
|
|
442
|
-
| `pdw.search` | vector, hybrid, byCategory, semantic | Search operations |
|
|
443
|
-
| `pdw.index` | add, search, save, load, getStats | HNSW index management |
|
|
444
|
-
| `pdw.ai` | embed, classify, shouldSave | AI/embedding operations |
|
|
445
|
-
| `pdw.graph` | extract, query | Knowledge graph |
|
|
446
|
-
|
|
447
|
-
### Supporting Namespaces
|
|
448
|
-
|
|
449
|
-
| Namespace | Methods | Description |
|
|
450
|
-
|-----------|---------|-------------|
|
|
451
|
-
| `pdw.encryption` | encrypt, decrypt | SEAL encryption |
|
|
452
|
-
| `pdw.capability` | create, verify | Capability management |
|
|
453
|
-
| `pdw.storage` | upload, download | Walrus storage |
|
|
454
|
-
| `pdw.wallet` | getAddress, getBalance | Wallet operations |
|
|
455
|
-
| `pdw.tx` | build, execute | Transaction building |
|
|
456
|
-
|
|
457
|
-
---
|
|
458
|
-
|
|
459
|
-
## Signer Architecture
|
|
460
|
-
|
|
461
|
-
The SDK supports multiple signing methods through the `UnifiedSigner` interface.
|
|
462
|
-
|
|
463
|
-
```text
|
|
464
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
465
|
-
│ Signer Architecture │
|
|
466
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
467
|
-
|
|
468
|
-
UnifiedSigner (Interface)
|
|
469
|
-
│
|
|
470
|
-
┌───────────────┼───────────────┐
|
|
471
|
-
│ │ │
|
|
472
|
-
▼ ▼ ▼
|
|
473
|
-
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────────┐
|
|
474
|
-
│ KeypairSigner │ │ WalletAdapter │ │ DappKitSigner │
|
|
475
|
-
│ │ │ Signer │ │ │
|
|
476
|
-
│ • Node.js │ │ • Deprecated │ │ • Browser (React) │
|
|
477
|
-
│ • Ed25519Keypair│ │ • Legacy wallet │ │ • @mysten/dapp-kit │
|
|
478
|
-
│ • CLI/Scripts │ │ standard │ │ • Slush, Sui Wallet │
|
|
479
|
-
│ • Testing │ │ │ │ • Popup signing │
|
|
480
|
-
└─────────────────┘ └─────────────────┘ └─────────────────────┘
|
|
481
|
-
|
|
482
|
-
UnifiedSigner Interface:
|
|
483
|
-
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
484
|
-
│ interface UnifiedSigner { │
|
|
485
|
-
│ getAddress(): string; │
|
|
486
|
-
│ getSigner(): Keypair | WalletAdapter | undefined; │
|
|
487
|
-
│ getClient(): SuiClient | null; // New in v0.6.1 │
|
|
488
|
-
│ signPersonalMessage(params): Promise<SignPersonalMessageResult>; │
|
|
489
|
-
│ signAndExecuteTransaction(params): Promise<SignAndExecuteResult>; │
|
|
490
|
-
│ } │
|
|
491
|
-
└─────────────────────────────────────────────────────────────────────────────┘
|
|
492
|
-
```
|
|
493
|
-
|
|
494
|
-
### Usage Examples
|
|
495
|
-
|
|
496
|
-
**Node.js with Keypair:**
|
|
497
|
-
```typescript
|
|
498
|
-
import { SimplePDWClient } from '@cmdoss/memwal-sdk';
|
|
499
|
-
import { Ed25519Keypair } from '@mysten/sui/keypairs/ed25519';
|
|
500
|
-
|
|
501
|
-
const keypair = Ed25519Keypair.fromSecretKey(secretKey);
|
|
502
|
-
const pdw = new SimplePDWClient({
|
|
503
|
-
signer: keypair,
|
|
504
|
-
network: 'testnet',
|
|
505
|
-
// ...
|
|
506
|
-
});
|
|
507
|
-
```
|
|
508
|
-
|
|
509
|
-
**Browser with DappKitSigner:**
|
|
510
|
-
```typescript
|
|
511
|
-
import { DappKitSigner, SimplePDWClient } from '@cmdoss/memwal-sdk/browser';
|
|
512
|
-
import { useSignAndExecuteTransaction, useSuiClient } from '@mysten/dapp-kit';
|
|
513
|
-
|
|
514
|
-
const signer = new DappKitSigner({
|
|
515
|
-
address: account.address,
|
|
516
|
-
client: suiClient,
|
|
517
|
-
signAndExecuteTransaction: async ({ transaction }) => {
|
|
518
|
-
const result = await signAndExecute({ transaction });
|
|
519
|
-
return { digest: result.digest, effects: result.effects };
|
|
520
|
-
},
|
|
521
|
-
});
|
|
522
|
-
|
|
523
|
-
const pdw = new SimplePDWClient({
|
|
524
|
-
signer,
|
|
525
|
-
network: 'testnet',
|
|
526
|
-
userAddress: account.address,
|
|
527
|
-
features: { enableLocalIndexing: false }, // For browser
|
|
528
|
-
});
|
|
529
|
-
|
|
530
|
-
// All SDK operations now trigger wallet popup for signing
|
|
531
|
-
await pdw.memory.create('Hello'); // → Wallet popup appears
|
|
532
|
-
```
|
|
533
|
-
|
|
534
|
-
### Entry Points
|
|
535
|
-
|
|
536
|
-
| Entry Point | Use Case | Node.js Deps |
|
|
537
|
-
|-------------|----------|--------------|
|
|
538
|
-
| `@cmdoss/memwal-sdk` | Node.js applications | Yes (hnswlib-node) |
|
|
539
|
-
| `@cmdoss/memwal-sdk/browser` | Browser applications | No |
|
|
540
|
-
|
|
541
|
-
---
|
|
542
|
-
|
|
543
|
-
## Related Documentation
|
|
544
|
-
|
|
545
|
-
- [README.md](./README.md) - Quick start guide
|
|
546
|
-
- [BENCHMARKS.md](./BENCHMARKS.md) - Performance metrics
|
|
547
|
-
- [CHANGELOG.md](./CHANGELOG.md) - Version history
|
|
1
|
+
# MemWal SDK Architecture
|
|
2
|
+
|
|
3
|
+
Detailed workflow diagrams and architecture documentation for MemWal (`@cmdoss/memwal-sdk`).
|
|
4
|
+
|
|
5
|
+
## Table of Contents
|
|
6
|
+
|
|
7
|
+
- [Architecture Overview](#architecture-overview)
|
|
8
|
+
- [Memory Creation Flow](#memory-creation-flow)
|
|
9
|
+
- [Search Flow](#search-flow)
|
|
10
|
+
- [Batch Upload (Quilt)](#batch-upload-quilt)
|
|
11
|
+
- [HNSW Vector Indexing](#hnsw-vector-indexing)
|
|
12
|
+
- [SEAL Encryption Flow](#seal-encryption-flow)
|
|
13
|
+
- [API Namespaces](#api-namespaces)
|
|
14
|
+
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
## Architecture Overview
|
|
18
|
+
|
|
19
|
+
```text
|
|
20
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
21
|
+
│ MemWal SDK Architecture │
|
|
22
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
23
|
+
|
|
24
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
25
|
+
│ SimplePDWClient │
|
|
26
|
+
│ ┌─────────────────────────────────────────────────────────────────────────┐│
|
|
27
|
+
│ │ Primary Namespaces ││
|
|
28
|
+
│ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ ││
|
|
29
|
+
│ │ │ memory │ │ search │ │ index │ │ ai │ │ graph │ ││
|
|
30
|
+
│ │ │ │ │ │ │ │ │ │ │ │ ││
|
|
31
|
+
│ │ │• create │ │• vector │ │• add │ │• embed │ │• extract │ ││
|
|
32
|
+
│ │ │• createBatch│• hybrid │ │• search │ │• classify│ │• query │ ││
|
|
33
|
+
│ │ │• get │ │• byCategory│ │• save │ │• shouldSave│ │ ││
|
|
34
|
+
│ │ │• delete │ │• semantic│ │• load │ │ │ │ │ ││
|
|
35
|
+
│ │ └──────────┘ └──────────┘ └──────────┘ └──────────┘ └──────────┘ ││
|
|
36
|
+
│ └─────────────────────────────────────────────────────────────────────────┘│
|
|
37
|
+
│ ┌─────────────────────────────────────────────────────────────────────────┐│
|
|
38
|
+
│ │ Service Container ││
|
|
39
|
+
│ │ embedding | memoryIndex | storage | tx | encryption | capability ││
|
|
40
|
+
│ └─────────────────────────────────────────────────────────────────────────┘│
|
|
41
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
42
|
+
│
|
|
43
|
+
┌───────────────┼───────────────┐
|
|
44
|
+
│ │ │
|
|
45
|
+
▼ ▼ ▼
|
|
46
|
+
┌─────────────────────┐ ┌─────────────────┐ ┌─────────────────────┐
|
|
47
|
+
│ Embedding API │ │ Sui Blockchain │ │ Walrus Storage │
|
|
48
|
+
│ │ │ │ │ │
|
|
49
|
+
│ • OpenRouter │ │ • Testnet │ │ • Publisher API │
|
|
50
|
+
│ • Gemini │ │ • Mainnet │ │ • Aggregator API │
|
|
51
|
+
│ • 3072 dimensions │ │ • Package ID │ │ • Quilt Batching │
|
|
52
|
+
└─────────────────────┘ └─────────────────┘ └─────────────────────┘
|
|
53
|
+
│
|
|
54
|
+
┌───────────────┼───────────────┐
|
|
55
|
+
│ │ │
|
|
56
|
+
▼ ▼ ▼
|
|
57
|
+
┌─────────────────────┐ ┌─────────────────┐ ┌─────────────────────┐
|
|
58
|
+
│ HNSW Vector Index │ │ SEAL Encryption│ │ Knowledge Graph │
|
|
59
|
+
│ │ │ │ │ │
|
|
60
|
+
│ • hnswlib-node │ │ • Threshold IBE │ │ • Entity extraction │
|
|
61
|
+
│ • hnswlib-wasm │ │ • 2-of-N decrypt│ │ • Relationships │
|
|
62
|
+
│ • Auto-detect env │ │ • Mysten SEAL │ │ • Gemini-powered │
|
|
63
|
+
└─────────────────────┘ └─────────────────┘ └─────────────────────┘
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
|
|
68
|
+
## Memory Creation Flow
|
|
69
|
+
|
|
70
|
+
```text
|
|
71
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
72
|
+
│ MemWal Memory Creation Workflow │
|
|
73
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
74
|
+
|
|
75
|
+
User Input: "I am working at CommandOSS as a software engineer"
|
|
76
|
+
│
|
|
77
|
+
▼
|
|
78
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
79
|
+
│ STEP 1: AI Pre-check (Optional) │
|
|
80
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
81
|
+
│ pdw.ai.shouldSave(content) → true/false │
|
|
82
|
+
│ │
|
|
83
|
+
│ • Analyzes content relevance │
|
|
84
|
+
│ • Filters noise/spam │
|
|
85
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
86
|
+
│
|
|
87
|
+
▼
|
|
88
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
89
|
+
│ STEP 2: AI Classification │
|
|
90
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
91
|
+
│ pdw.ai.classify(content) → { category, importance, topic, summary } │
|
|
92
|
+
│ │
|
|
93
|
+
│ Output: { category: "fact", importance: 8, topic: "employment" } │
|
|
94
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
95
|
+
│
|
|
96
|
+
▼
|
|
97
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
98
|
+
│ STEP 3: Embedding Generation │
|
|
99
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
100
|
+
│ pdw.ai.embed(content) → Float32Array[3072] │
|
|
101
|
+
│ │
|
|
102
|
+
│ • OpenRouter or Gemini API │
|
|
103
|
+
│ • 3072 dimensions (text-embedding-3-large) │
|
|
104
|
+
│ • Normalized vector for cosine similarity │
|
|
105
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
106
|
+
│
|
|
107
|
+
▼
|
|
108
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
109
|
+
│ STEP 4: Walrus Upload │
|
|
110
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
111
|
+
│ Memory Package uploaded to Walrus: │
|
|
112
|
+
│ { │
|
|
113
|
+
│ content: "I am working at CommandOSS...", │
|
|
114
|
+
│ embedding: [0.0234, -0.0156, ...], // 3072 dims │
|
|
115
|
+
│ metadata: { category, importance, topic }, │
|
|
116
|
+
│ timestamp: 1701705600000, │
|
|
117
|
+
│ identity: "0xb59f00b2454bef14d538b..." │
|
|
118
|
+
│ } │
|
|
119
|
+
│ │
|
|
120
|
+
│ Output: blobId = "KEG_kj_5nx8wr3eJFZwIkJtjKvEGaUNMvTvhDYddaPU" │
|
|
121
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
122
|
+
│
|
|
123
|
+
▼
|
|
124
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
125
|
+
│ STEP 5: On-chain Registration (Sui Blockchain) │
|
|
126
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
127
|
+
│ Transaction: memory::create_memory_record │
|
|
128
|
+
│ │
|
|
129
|
+
│ Parameters: │
|
|
130
|
+
│ • blob_id: "KEG_kj_5nx8wr3eJFZwIkJtjKvEGaUNMvTvhDYddaPU" │
|
|
131
|
+
│ • category: "fact" │
|
|
132
|
+
│ • importance: 8 │
|
|
133
|
+
│ • content_hash: keccak256(content) │
|
|
134
|
+
│ │
|
|
135
|
+
│ Output: MemoryRecord { id: "0x6bce7d4140b7e6572...", owner: "0xb59f..." } │
|
|
136
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
137
|
+
│
|
|
138
|
+
▼
|
|
139
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
140
|
+
│ STEP 6: Local HNSW Indexing │
|
|
141
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
142
|
+
│ Auto-detected implementation: │
|
|
143
|
+
│ • Node.js: hnswlib-node (native C++, fastest) │
|
|
144
|
+
│ • Browser: hnswlib-wasm (WebAssembly fallback) │
|
|
145
|
+
│ │
|
|
146
|
+
│ Index entry (NO content stored for privacy): │
|
|
147
|
+
│ { memoryId, blobId, category, importance, timestamp, vectorId } │
|
|
148
|
+
│ │
|
|
149
|
+
│ Persistence: │
|
|
150
|
+
│ • Node.js: Filesystem (.pdw-indexes/) │
|
|
151
|
+
│ • Browser: IndexedDB │
|
|
152
|
+
│ • Cloud: Optional Walrus backup │
|
|
153
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
### Usage
|
|
157
|
+
|
|
158
|
+
```typescript
|
|
159
|
+
import { SimplePDWClient } from '@cmdoss/memwal-sdk';
|
|
160
|
+
|
|
161
|
+
const pdw = new SimplePDWClient({
|
|
162
|
+
signer: keypair,
|
|
163
|
+
network: 'testnet',
|
|
164
|
+
packageId: '0x...',
|
|
165
|
+
embedding: { provider: 'openrouter', apiKey: '...' }
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
await pdw.ready();
|
|
169
|
+
|
|
170
|
+
// Create single memory
|
|
171
|
+
const memory = await pdw.memory.create('I work at CommandOSS', {
|
|
172
|
+
category: 'fact',
|
|
173
|
+
importance: 8
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
// Create batch (uses Quilt for ~90% gas savings)
|
|
177
|
+
const memories = await pdw.memory.createBatch([
|
|
178
|
+
'Memory 1 content',
|
|
179
|
+
'Memory 2 content',
|
|
180
|
+
'Memory 3 content'
|
|
181
|
+
]);
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
## Search Flow
|
|
187
|
+
|
|
188
|
+
```text
|
|
189
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
190
|
+
│ MemWal Search Workflow │
|
|
191
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
192
|
+
|
|
193
|
+
Query: "What company do I work for?"
|
|
194
|
+
│
|
|
195
|
+
▼
|
|
196
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
197
|
+
│ STEP 1: Query Embedding │
|
|
198
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
199
|
+
│ pdw.ai.embed("What company do I work for?") │
|
|
200
|
+
│ Output: Float32Array[3072] │
|
|
201
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
202
|
+
│
|
|
203
|
+
▼
|
|
204
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
205
|
+
│ STEP 2: HNSW Search (Local Index) │
|
|
206
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
207
|
+
│ Algorithm: Hierarchical Navigable Small World │
|
|
208
|
+
│ Complexity: O(log n) │
|
|
209
|
+
│ │
|
|
210
|
+
│ Returns top-k nearest neighbors by cosine similarity: │
|
|
211
|
+
│ ┌────────┬────────────────────────────────────────────┬───────────┐ │
|
|
212
|
+
│ │ Rank │ Memory ID │ Score │ │
|
|
213
|
+
│ ├────────┼────────────────────────────────────────────┼───────────┤ │
|
|
214
|
+
│ │ 1 │ 0x6bce7d4140b7e6572df79aa1c8b12abfb... │ 0.8542 │ │
|
|
215
|
+
│ │ 2 │ 0xac333f3c2de375da1fbbee1bdada24ff... │ 0.7231 │ │
|
|
216
|
+
│ │ 3 │ 0xbb7a63c4a7a62d2a0b083bc02ee96593... │ 0.6890 │ │
|
|
217
|
+
│ └────────┴────────────────────────────────────────────┴───────────┘ │
|
|
218
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
219
|
+
│
|
|
220
|
+
▼
|
|
221
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
222
|
+
│ STEP 3: Content Retrieval (Walrus) │
|
|
223
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
224
|
+
│ For each result, fetch content from Walrus: │
|
|
225
|
+
│ │
|
|
226
|
+
│ • Download MemoryPackage from blobId │
|
|
227
|
+
│ • Decrypt if encrypted (SEAL) │
|
|
228
|
+
│ • Cache locally for subsequent access │
|
|
229
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
230
|
+
│
|
|
231
|
+
▼
|
|
232
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
233
|
+
│ RESULT │
|
|
234
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
235
|
+
│ [ │
|
|
236
|
+
│ { │
|
|
237
|
+
│ id: "0x6bce7d4140b7e6572df79aa1c8b12abfb...", │
|
|
238
|
+
│ content: "I am working at CommandOSS as a software engineer", │
|
|
239
|
+
│ score: 0.8542, │
|
|
240
|
+
│ category: "fact", │
|
|
241
|
+
│ importance: 8 │
|
|
242
|
+
│ }, │
|
|
243
|
+
│ ... │
|
|
244
|
+
│ ] │
|
|
245
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
### Search Methods
|
|
249
|
+
|
|
250
|
+
```typescript
|
|
251
|
+
// Vector search (semantic)
|
|
252
|
+
const results = await pdw.search.vector('work experience', { limit: 5 });
|
|
253
|
+
|
|
254
|
+
// Filter by category
|
|
255
|
+
const facts = await pdw.search.byCategory('fact', { limit: 10 });
|
|
256
|
+
|
|
257
|
+
// Hybrid search (vector + category filter)
|
|
258
|
+
const filtered = await pdw.search.hybrid('work', { category: 'fact' });
|
|
259
|
+
|
|
260
|
+
// With content fetched
|
|
261
|
+
const withContent = await pdw.search.withContent(results);
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
---
|
|
265
|
+
|
|
266
|
+
## Batch Upload (Quilt)
|
|
267
|
+
|
|
268
|
+
Walrus Quilt enables batch uploads with ~90% gas savings.
|
|
269
|
+
|
|
270
|
+
```text
|
|
271
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
272
|
+
│ Quilt Batch Upload Flow │
|
|
273
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
274
|
+
|
|
275
|
+
Individual Upload: Quilt Batch Upload:
|
|
276
|
+
┌─────────────┐ ┌─────────────┐
|
|
277
|
+
│ Memory 1 │──► Transaction 1 │ Memory 1 │
|
|
278
|
+
└─────────────┘ │ Memory 2 │──► Single Transaction
|
|
279
|
+
┌─────────────┐ │ Memory 3 │ (Quilt)
|
|
280
|
+
│ Memory 2 │──► Transaction 2 │ Memory 4 │
|
|
281
|
+
└─────────────┘ │ Memory 5 │
|
|
282
|
+
┌─────────────┐ └─────────────┘
|
|
283
|
+
│ Memory 3 │──► Transaction 3
|
|
284
|
+
└─────────────┘ Gas: 1x base fee
|
|
285
|
+
vs
|
|
286
|
+
Gas: 3x base fee Individual: 5x base fee
|
|
287
|
+
Savings: ~90%
|
|
288
|
+
|
|
289
|
+
Quilt Structure:
|
|
290
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
291
|
+
│ quiltId: "abc123..." │
|
|
292
|
+
│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │
|
|
293
|
+
│ │ quiltPatchId: 1 │ │ quiltPatchId: 2 │ │ quiltPatchId: 3 │ │
|
|
294
|
+
│ │ identifier: m1 │ │ identifier: m2 │ │ identifier: m3 │ │
|
|
295
|
+
│ │ tags: {...} │ │ tags: {...} │ │ tags: {...} │ │
|
|
296
|
+
│ │ content: ... │ │ content: ... │ │ content: ... │ │
|
|
297
|
+
│ └─────────────────┘ └─────────────────┘ └─────────────────┘ │
|
|
298
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
### Usage
|
|
302
|
+
|
|
303
|
+
```typescript
|
|
304
|
+
// Batch create memories (automatically uses Quilt)
|
|
305
|
+
const results = await pdw.memory.createBatch([
|
|
306
|
+
'First memory content',
|
|
307
|
+
'Second memory content',
|
|
308
|
+
'Third memory content'
|
|
309
|
+
]);
|
|
310
|
+
|
|
311
|
+
// Results include quilt information
|
|
312
|
+
results.forEach(r => {
|
|
313
|
+
console.log(`ID: ${r.id}, BlobId: ${r.blobId}`);
|
|
314
|
+
});
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
---
|
|
318
|
+
|
|
319
|
+
## HNSW Vector Indexing
|
|
320
|
+
|
|
321
|
+
The SDK auto-detects the environment and uses the optimal HNSW implementation.
|
|
322
|
+
|
|
323
|
+
```text
|
|
324
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
325
|
+
│ Environment-Aware HNSW Factory │
|
|
326
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
327
|
+
|
|
328
|
+
createHnswService()
|
|
329
|
+
│
|
|
330
|
+
┌─────────────┴─────────────┐
|
|
331
|
+
│ │
|
|
332
|
+
isBrowser()? isNode()?
|
|
333
|
+
│ │
|
|
334
|
+
▼ ▼
|
|
335
|
+
┌─────────────────────────┐ ┌─────────────────────────┐
|
|
336
|
+
│ BrowserHnswIndexService│ │ NodeHnswService │
|
|
337
|
+
│ (hnswlib-wasm) │ │ (hnswlib-node) │
|
|
338
|
+
│ │ │ │
|
|
339
|
+
│ • WebAssembly │ │ • Native C++ bindings │
|
|
340
|
+
│ • IndexedDB persistence│ │ • Filesystem persistence│
|
|
341
|
+
│ • ~50% slower than node│ │ • Fastest performance │
|
|
342
|
+
│ • Universal fallback │ │ • Requires build tools │
|
|
343
|
+
└─────────────────────────┘ └─────────────────────────┘
|
|
344
|
+
|
|
345
|
+
Singleton Pattern:
|
|
346
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
347
|
+
│ First call: Creates new instance and initializes │
|
|
348
|
+
│ Subsequent calls: Returns existing singleton │
|
|
349
|
+
│ │
|
|
350
|
+
│ Benefits: │
|
|
351
|
+
│ • Prevents redundant index loading │
|
|
352
|
+
│ • Shared index across all SDK clients │
|
|
353
|
+
│ • Automatic environment detection │
|
|
354
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
### Index Configuration
|
|
358
|
+
|
|
359
|
+
```typescript
|
|
360
|
+
// Default configuration
|
|
361
|
+
const indexConfig = {
|
|
362
|
+
dimension: 3072, // Embedding dimensions
|
|
363
|
+
maxElements: 10000, // Maximum vectors
|
|
364
|
+
efConstruction: 200, // Build-time quality
|
|
365
|
+
m: 16, // Connections per layer
|
|
366
|
+
spaceType: 'cosine' // Distance metric
|
|
367
|
+
};
|
|
368
|
+
|
|
369
|
+
// Access via namespace
|
|
370
|
+
const stats = await pdw.index.getStats();
|
|
371
|
+
// { currentCount: 150, dimension: 3072, maxElements: 10000 }
|
|
372
|
+
|
|
373
|
+
// Manual save/load
|
|
374
|
+
await pdw.index.save(); // Save to Walrus for backup
|
|
375
|
+
await pdw.index.load(); // Load from Walrus backup
|
|
376
|
+
```
|
|
377
|
+
|
|
378
|
+
---
|
|
379
|
+
|
|
380
|
+
## SEAL Encryption Flow
|
|
381
|
+
|
|
382
|
+
```text
|
|
383
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
384
|
+
│ SEAL Encryption Workflow │
|
|
385
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
386
|
+
|
|
387
|
+
ENCRYPTION
|
|
388
|
+
│
|
|
389
|
+
▼
|
|
390
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
391
|
+
│ STEP 1: Create MemoryCap (On-chain) │
|
|
392
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
393
|
+
│ Creates on-chain capability object for key derivation │
|
|
394
|
+
│ │
|
|
395
|
+
│ MemoryCap { │
|
|
396
|
+
│ id: "0x05ac7bdc83308ea2cea429afc9fd6bbc91838b...", │
|
|
397
|
+
│ owner: "0xb59f00b2454bef14d538b...", │
|
|
398
|
+
│ nonce: [random 32 bytes] ← Used for key derivation │
|
|
399
|
+
│ } │
|
|
400
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
401
|
+
│
|
|
402
|
+
▼
|
|
403
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
404
|
+
│ STEP 2: SEAL Encrypt │
|
|
405
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
406
|
+
│ • Identity-Based Encryption (IBE) │
|
|
407
|
+
│ • Threshold: 2 of N key servers must agree │
|
|
408
|
+
│ • Key servers: Mysten Labs operated │
|
|
409
|
+
│ │
|
|
410
|
+
│ Output: { encryptedObject, keyId, nonce } │
|
|
411
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
412
|
+
│
|
|
413
|
+
▼
|
|
414
|
+
Upload to Walrus
|
|
415
|
+
│
|
|
416
|
+
DECRYPTION
|
|
417
|
+
│
|
|
418
|
+
▼
|
|
419
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
420
|
+
│ STEP 3: Request Decryption Key │
|
|
421
|
+
├─────────────────────────────────────────────────────────────────────────────┤
|
|
422
|
+
│ 1. Build seal_approve transaction with MemoryCap │
|
|
423
|
+
│ 2. Key servers verify: │
|
|
424
|
+
│ • Caller owns the MemoryCap │
|
|
425
|
+
│ • key_id matches MemoryCap's derived key │
|
|
426
|
+
│ 3. If verified, key servers release decryption shares │
|
|
427
|
+
│ 4. Combine shares (threshold) to decrypt │
|
|
428
|
+
│ │
|
|
429
|
+
│ Output: Decrypted content │
|
|
430
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
---
|
|
434
|
+
|
|
435
|
+
## API Namespaces
|
|
436
|
+
|
|
437
|
+
### Primary Namespaces
|
|
438
|
+
|
|
439
|
+
| Namespace | Methods | Description |
|
|
440
|
+
|-----------|---------|-------------|
|
|
441
|
+
| `pdw.memory` | create, createBatch, get, delete, list | Memory CRUD operations |
|
|
442
|
+
| `pdw.search` | vector, hybrid, byCategory, semantic | Search operations |
|
|
443
|
+
| `pdw.index` | add, search, save, load, getStats | HNSW index management |
|
|
444
|
+
| `pdw.ai` | embed, classify, shouldSave | AI/embedding operations |
|
|
445
|
+
| `pdw.graph` | extract, query | Knowledge graph |
|
|
446
|
+
|
|
447
|
+
### Supporting Namespaces
|
|
448
|
+
|
|
449
|
+
| Namespace | Methods | Description |
|
|
450
|
+
|-----------|---------|-------------|
|
|
451
|
+
| `pdw.encryption` | encrypt, decrypt | SEAL encryption |
|
|
452
|
+
| `pdw.capability` | create, verify | Capability management |
|
|
453
|
+
| `pdw.storage` | upload, download | Walrus storage |
|
|
454
|
+
| `pdw.wallet` | getAddress, getBalance | Wallet operations |
|
|
455
|
+
| `pdw.tx` | build, execute | Transaction building |
|
|
456
|
+
|
|
457
|
+
---
|
|
458
|
+
|
|
459
|
+
## Signer Architecture
|
|
460
|
+
|
|
461
|
+
The SDK supports multiple signing methods through the `UnifiedSigner` interface.
|
|
462
|
+
|
|
463
|
+
```text
|
|
464
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
465
|
+
│ Signer Architecture │
|
|
466
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
467
|
+
|
|
468
|
+
UnifiedSigner (Interface)
|
|
469
|
+
│
|
|
470
|
+
┌───────────────┼───────────────┐
|
|
471
|
+
│ │ │
|
|
472
|
+
▼ ▼ ▼
|
|
473
|
+
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────────┐
|
|
474
|
+
│ KeypairSigner │ │ WalletAdapter │ │ DappKitSigner │
|
|
475
|
+
│ │ │ Signer │ │ │
|
|
476
|
+
│ • Node.js │ │ • Deprecated │ │ • Browser (React) │
|
|
477
|
+
│ • Ed25519Keypair│ │ • Legacy wallet │ │ • @mysten/dapp-kit │
|
|
478
|
+
│ • CLI/Scripts │ │ standard │ │ • Slush, Sui Wallet │
|
|
479
|
+
│ • Testing │ │ │ │ • Popup signing │
|
|
480
|
+
└─────────────────┘ └─────────────────┘ └─────────────────────┘
|
|
481
|
+
|
|
482
|
+
UnifiedSigner Interface:
|
|
483
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
484
|
+
│ interface UnifiedSigner { │
|
|
485
|
+
│ getAddress(): string; │
|
|
486
|
+
│ getSigner(): Keypair | WalletAdapter | undefined; │
|
|
487
|
+
│ getClient(): SuiClient | null; // New in v0.6.1 │
|
|
488
|
+
│ signPersonalMessage(params): Promise<SignPersonalMessageResult>; │
|
|
489
|
+
│ signAndExecuteTransaction(params): Promise<SignAndExecuteResult>; │
|
|
490
|
+
│ } │
|
|
491
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
492
|
+
```
|
|
493
|
+
|
|
494
|
+
### Usage Examples
|
|
495
|
+
|
|
496
|
+
**Node.js with Keypair:**
|
|
497
|
+
```typescript
|
|
498
|
+
import { SimplePDWClient } from '@cmdoss/memwal-sdk';
|
|
499
|
+
import { Ed25519Keypair } from '@mysten/sui/keypairs/ed25519';
|
|
500
|
+
|
|
501
|
+
const keypair = Ed25519Keypair.fromSecretKey(secretKey);
|
|
502
|
+
const pdw = new SimplePDWClient({
|
|
503
|
+
signer: keypair,
|
|
504
|
+
network: 'testnet',
|
|
505
|
+
// ...
|
|
506
|
+
});
|
|
507
|
+
```
|
|
508
|
+
|
|
509
|
+
**Browser with DappKitSigner:**
|
|
510
|
+
```typescript
|
|
511
|
+
import { DappKitSigner, SimplePDWClient } from '@cmdoss/memwal-sdk/browser';
|
|
512
|
+
import { useSignAndExecuteTransaction, useSuiClient } from '@mysten/dapp-kit';
|
|
513
|
+
|
|
514
|
+
const signer = new DappKitSigner({
|
|
515
|
+
address: account.address,
|
|
516
|
+
client: suiClient,
|
|
517
|
+
signAndExecuteTransaction: async ({ transaction }) => {
|
|
518
|
+
const result = await signAndExecute({ transaction });
|
|
519
|
+
return { digest: result.digest, effects: result.effects };
|
|
520
|
+
},
|
|
521
|
+
});
|
|
522
|
+
|
|
523
|
+
const pdw = new SimplePDWClient({
|
|
524
|
+
signer,
|
|
525
|
+
network: 'testnet',
|
|
526
|
+
userAddress: account.address,
|
|
527
|
+
features: { enableLocalIndexing: false }, // For browser
|
|
528
|
+
});
|
|
529
|
+
|
|
530
|
+
// All SDK operations now trigger wallet popup for signing
|
|
531
|
+
await pdw.memory.create('Hello'); // → Wallet popup appears
|
|
532
|
+
```
|
|
533
|
+
|
|
534
|
+
### Entry Points
|
|
535
|
+
|
|
536
|
+
| Entry Point | Use Case | Node.js Deps |
|
|
537
|
+
|-------------|----------|--------------|
|
|
538
|
+
| `@cmdoss/memwal-sdk` | Node.js applications | Yes (hnswlib-node) |
|
|
539
|
+
| `@cmdoss/memwal-sdk/browser` | Browser applications | No |
|
|
540
|
+
|
|
541
|
+
---
|
|
542
|
+
|
|
543
|
+
## Related Documentation
|
|
544
|
+
|
|
545
|
+
- [README.md](./README.md) - Quick start guide
|
|
546
|
+
- [BENCHMARKS.md](./BENCHMARKS.md) - Performance metrics
|
|
547
|
+
- [CHANGELOG.md](./CHANGELOG.md) - Version history
|