@soulcraft/brainy 3.50.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +201 -0
- package/README.md +358 -658
- package/dist/api/ConfigAPI.js +56 -19
- package/dist/api/DataAPI.js +24 -18
- package/dist/augmentations/storageAugmentations.d.ts +24 -0
- package/dist/augmentations/storageAugmentations.js +22 -0
- package/dist/brainy.js +32 -9
- package/dist/cli/commands/core.d.ts +20 -10
- package/dist/cli/commands/core.js +384 -82
- package/dist/cli/commands/import.d.ts +41 -0
- package/dist/cli/commands/import.js +456 -0
- package/dist/cli/commands/insights.d.ts +34 -0
- package/dist/cli/commands/insights.js +300 -0
- package/dist/cli/commands/neural.d.ts +6 -12
- package/dist/cli/commands/neural.js +113 -10
- package/dist/cli/commands/nlp.d.ts +28 -0
- package/dist/cli/commands/nlp.js +246 -0
- package/dist/cli/commands/storage.d.ts +64 -0
- package/dist/cli/commands/storage.js +730 -0
- package/dist/cli/index.js +210 -24
- package/dist/coreTypes.d.ts +206 -34
- package/dist/distributed/configManager.js +8 -6
- package/dist/distributed/shardMigration.js +2 -0
- package/dist/distributed/storageDiscovery.js +6 -4
- package/dist/embeddings/EmbeddingManager.d.ts +2 -2
- package/dist/embeddings/EmbeddingManager.js +5 -1
- package/dist/graph/lsm/LSMTree.js +32 -20
- package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
- package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
- package/dist/storage/adapters/azureBlobStorage.js +1809 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
- package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
- package/dist/storage/adapters/fileSystemStorage.js +204 -127
- package/dist/storage/adapters/gcsStorage.d.ts +119 -9
- package/dist/storage/adapters/gcsStorage.js +317 -62
- package/dist/storage/adapters/memoryStorage.d.ts +30 -18
- package/dist/storage/adapters/memoryStorage.js +99 -94
- package/dist/storage/adapters/opfsStorage.d.ts +48 -10
- package/dist/storage/adapters/opfsStorage.js +201 -80
- package/dist/storage/adapters/r2Storage.d.ts +12 -5
- package/dist/storage/adapters/r2Storage.js +63 -15
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
- package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
- package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
- package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
- package/dist/storage/baseStorage.d.ts +41 -38
- package/dist/storage/baseStorage.js +110 -134
- package/dist/storage/storageFactory.d.ts +29 -2
- package/dist/storage/storageFactory.js +30 -1
- package/dist/utils/entityIdMapper.js +5 -2
- package/dist/utils/fieldTypeInference.js +8 -1
- package/dist/utils/metadataFilter.d.ts +3 -2
- package/dist/utils/metadataFilter.js +1 -0
- package/dist/utils/metadataIndex.js +2 -0
- package/dist/utils/metadataIndexChunking.js +9 -4
- package/dist/utils/periodicCleanup.js +1 -0
- package/package.json +3 -1
|
@@ -2,9 +2,8 @@
|
|
|
2
2
|
* Memory Storage Adapter
|
|
3
3
|
* In-memory storage adapter for environments where persistent storage is not available or needed
|
|
4
4
|
*/
|
|
5
|
-
import {
|
|
5
|
+
import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
|
|
6
6
|
import { BaseStorage } from '../baseStorage.js';
|
|
7
|
-
import { PaginatedResult } from '../../types/paginationTypes.js';
|
|
8
7
|
/**
|
|
9
8
|
* In-memory storage adapter
|
|
10
9
|
* Uses Maps to store data in memory
|
|
@@ -24,18 +23,19 @@ export declare class MemoryStorage extends BaseStorage {
|
|
|
24
23
|
*/
|
|
25
24
|
init(): Promise<void>;
|
|
26
25
|
/**
|
|
27
|
-
* Save a noun to storage
|
|
26
|
+
* Save a noun to storage (v4.0.0: pure vector only, no metadata)
|
|
28
27
|
*/
|
|
29
28
|
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
|
30
29
|
/**
|
|
31
|
-
* Get a noun from storage (
|
|
32
|
-
*
|
|
30
|
+
* Get a noun from storage (v4.0.0: returns pure vector only)
|
|
31
|
+
* Base class handles combining with metadata
|
|
33
32
|
*/
|
|
34
33
|
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
|
35
34
|
/**
|
|
36
35
|
* Get nouns with pagination and filtering
|
|
36
|
+
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
|
|
37
37
|
* @param options Pagination and filtering options
|
|
38
|
-
* @returns Promise that resolves to a paginated result of nouns
|
|
38
|
+
* @returns Promise that resolves to a paginated result of nouns with metadata
|
|
39
39
|
*/
|
|
40
40
|
getNouns(options?: {
|
|
41
41
|
pagination?: {
|
|
@@ -48,16 +48,22 @@ export declare class MemoryStorage extends BaseStorage {
|
|
|
48
48
|
service?: string | string[];
|
|
49
49
|
metadata?: Record<string, any>;
|
|
50
50
|
};
|
|
51
|
-
}): Promise<
|
|
51
|
+
}): Promise<{
|
|
52
|
+
items: HNSWNounWithMetadata[];
|
|
53
|
+
totalCount?: number;
|
|
54
|
+
hasMore: boolean;
|
|
55
|
+
nextCursor?: string;
|
|
56
|
+
}>;
|
|
52
57
|
/**
|
|
53
58
|
* Get nouns with pagination - simplified interface for compatibility
|
|
59
|
+
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
|
|
54
60
|
*/
|
|
55
61
|
getNounsWithPagination(options?: {
|
|
56
62
|
limit?: number;
|
|
57
63
|
cursor?: string;
|
|
58
64
|
filter?: any;
|
|
59
65
|
}): Promise<{
|
|
60
|
-
items:
|
|
66
|
+
items: HNSWNounWithMetadata[];
|
|
61
67
|
totalCount: number;
|
|
62
68
|
hasMore: boolean;
|
|
63
69
|
nextCursor?: string;
|
|
@@ -70,22 +76,23 @@ export declare class MemoryStorage extends BaseStorage {
|
|
|
70
76
|
*/
|
|
71
77
|
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
|
72
78
|
/**
|
|
73
|
-
* Delete a noun from storage
|
|
79
|
+
* Delete a noun from storage (v4.0.0)
|
|
74
80
|
*/
|
|
75
81
|
protected deleteNoun_internal(id: string): Promise<void>;
|
|
76
82
|
/**
|
|
77
|
-
* Save a verb to storage
|
|
83
|
+
* Save a verb to storage (v4.0.0: pure vector + core fields, no metadata)
|
|
78
84
|
*/
|
|
79
85
|
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
|
80
86
|
/**
|
|
81
|
-
* Get a verb from storage (
|
|
82
|
-
*
|
|
87
|
+
* Get a verb from storage (v4.0.0: returns pure vector + core fields)
|
|
88
|
+
* Base class handles combining with metadata
|
|
83
89
|
*/
|
|
84
90
|
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
|
85
91
|
/**
|
|
86
92
|
* Get verbs with pagination and filtering
|
|
93
|
+
* v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field)
|
|
87
94
|
* @param options Pagination and filtering options
|
|
88
|
-
* @returns Promise that resolves to a paginated result of verbs
|
|
95
|
+
* @returns Promise that resolves to a paginated result of verbs with metadata
|
|
89
96
|
*/
|
|
90
97
|
getVerbs(options?: {
|
|
91
98
|
pagination?: {
|
|
@@ -100,22 +107,27 @@ export declare class MemoryStorage extends BaseStorage {
|
|
|
100
107
|
service?: string | string[];
|
|
101
108
|
metadata?: Record<string, any>;
|
|
102
109
|
};
|
|
103
|
-
}): Promise<
|
|
110
|
+
}): Promise<{
|
|
111
|
+
items: HNSWVerbWithMetadata[];
|
|
112
|
+
totalCount?: number;
|
|
113
|
+
hasMore: boolean;
|
|
114
|
+
nextCursor?: string;
|
|
115
|
+
}>;
|
|
104
116
|
/**
|
|
105
117
|
* Get verbs by source
|
|
106
118
|
* @deprecated Use getVerbs() with filter.sourceId instead
|
|
107
119
|
*/
|
|
108
|
-
protected getVerbsBySource_internal(sourceId: string): Promise<
|
|
120
|
+
protected getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
109
121
|
/**
|
|
110
122
|
* Get verbs by target
|
|
111
123
|
* @deprecated Use getVerbs() with filter.targetId instead
|
|
112
124
|
*/
|
|
113
|
-
protected getVerbsByTarget_internal(targetId: string): Promise<
|
|
125
|
+
protected getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
114
126
|
/**
|
|
115
127
|
* Get verbs by type
|
|
116
128
|
* @deprecated Use getVerbs() with filter.verbType instead
|
|
117
129
|
*/
|
|
118
|
-
protected getVerbsByType_internal(type: string): Promise<
|
|
130
|
+
protected getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]>;
|
|
119
131
|
/**
|
|
120
132
|
* Delete a verb from storage
|
|
121
133
|
*/
|
|
@@ -169,7 +181,7 @@ export declare class MemoryStorage extends BaseStorage {
|
|
|
169
181
|
*/
|
|
170
182
|
protected getStatisticsData(): Promise<StatisticsData | null>;
|
|
171
183
|
/**
|
|
172
|
-
* Initialize counts from in-memory storage - O(1) operation
|
|
184
|
+
* Initialize counts from in-memory storage - O(1) operation (v4.0.0)
|
|
173
185
|
*/
|
|
174
186
|
protected initializeCounts(): Promise<void>;
|
|
175
187
|
/**
|
|
@@ -36,19 +36,19 @@ export class MemoryStorage extends BaseStorage {
|
|
|
36
36
|
this.isInitialized = true;
|
|
37
37
|
}
|
|
38
38
|
/**
|
|
39
|
-
* Save a noun to storage
|
|
39
|
+
* Save a noun to storage (v4.0.0: pure vector only, no metadata)
|
|
40
40
|
*/
|
|
41
41
|
async saveNoun_internal(noun) {
|
|
42
42
|
const isNew = !this.nouns.has(noun.id);
|
|
43
43
|
// Create a deep copy to avoid reference issues
|
|
44
|
-
//
|
|
45
|
-
// Metadata is saved separately via saveNounMetadata()
|
|
44
|
+
// v4.0.0: Store ONLY vector data (no metadata field)
|
|
45
|
+
// Metadata is saved separately via saveNounMetadata() by base class
|
|
46
46
|
const nounCopy = {
|
|
47
47
|
id: noun.id,
|
|
48
48
|
vector: [...noun.vector],
|
|
49
49
|
connections: new Map(),
|
|
50
50
|
level: noun.level || 0
|
|
51
|
-
// NO metadata field
|
|
51
|
+
// ✅ NO metadata field in v4.0.0
|
|
52
52
|
};
|
|
53
53
|
// Copy connections
|
|
54
54
|
for (const [level, connections] of noun.connections.entries()) {
|
|
@@ -56,15 +56,11 @@ export class MemoryStorage extends BaseStorage {
|
|
|
56
56
|
}
|
|
57
57
|
// Save the noun directly in the nouns map
|
|
58
58
|
this.nouns.set(noun.id, nounCopy);
|
|
59
|
-
//
|
|
60
|
-
if (isNew) {
|
|
61
|
-
const type = noun.metadata?.type || noun.metadata?.nounType || 'default';
|
|
62
|
-
this.incrementEntityCount(type);
|
|
63
|
-
}
|
|
59
|
+
// Note: Count tracking happens in saveNounMetadata since type info is in metadata now
|
|
64
60
|
}
|
|
65
61
|
/**
|
|
66
|
-
* Get a noun from storage (
|
|
67
|
-
*
|
|
62
|
+
* Get a noun from storage (v4.0.0: returns pure vector only)
|
|
63
|
+
* Base class handles combining with metadata
|
|
68
64
|
*/
|
|
69
65
|
async getNoun_internal(id) {
|
|
70
66
|
// Get the noun directly from the nouns map
|
|
@@ -74,28 +70,25 @@ export class MemoryStorage extends BaseStorage {
|
|
|
74
70
|
return null;
|
|
75
71
|
}
|
|
76
72
|
// Return a deep copy to avoid reference issues
|
|
73
|
+
// v4.0.0: Return ONLY vector data (no metadata field)
|
|
77
74
|
const nounCopy = {
|
|
78
75
|
id: noun.id,
|
|
79
76
|
vector: [...noun.vector],
|
|
80
77
|
connections: new Map(),
|
|
81
78
|
level: noun.level || 0
|
|
79
|
+
// ✅ NO metadata field in v4.0.0
|
|
82
80
|
};
|
|
83
81
|
// Copy connections
|
|
84
82
|
for (const [level, connections] of noun.connections.entries()) {
|
|
85
83
|
nounCopy.connections.set(level, new Set(connections));
|
|
86
84
|
}
|
|
87
|
-
|
|
88
|
-
const metadata = await this.getNounMetadata(id);
|
|
89
|
-
// Combine into complete noun object
|
|
90
|
-
return {
|
|
91
|
-
...nounCopy,
|
|
92
|
-
metadata: metadata || {}
|
|
93
|
-
};
|
|
85
|
+
return nounCopy;
|
|
94
86
|
}
|
|
95
87
|
/**
|
|
96
88
|
* Get nouns with pagination and filtering
|
|
89
|
+
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
|
|
97
90
|
* @param options Pagination and filtering options
|
|
98
|
-
* @returns Promise that resolves to a paginated result of nouns
|
|
91
|
+
* @returns Promise that resolves to a paginated result of nouns with metadata
|
|
99
92
|
*/
|
|
100
93
|
async getNouns(options = {}) {
|
|
101
94
|
const pagination = options.pagination || {};
|
|
@@ -113,13 +106,14 @@ export class MemoryStorage extends BaseStorage {
|
|
|
113
106
|
// First, collect all noun IDs that match the filter criteria
|
|
114
107
|
const matchingIds = [];
|
|
115
108
|
// Iterate through all nouns to find matches
|
|
109
|
+
// v4.0.0: Load metadata from separate storage (no embedded metadata field)
|
|
116
110
|
for (const [nounId, noun] of this.nouns.entries()) {
|
|
117
|
-
//
|
|
118
|
-
const
|
|
119
|
-
//
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
111
|
+
// Get metadata from separate storage
|
|
112
|
+
const metadata = await this.getNounMetadata(nounId);
|
|
113
|
+
// Skip if no metadata (shouldn't happen in v4.0.0 but be defensive)
|
|
114
|
+
if (!metadata) {
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
123
117
|
// Filter by noun type if specified
|
|
124
118
|
if (nounTypes && metadata.noun && !nounTypes.includes(metadata.noun)) {
|
|
125
119
|
continue;
|
|
@@ -150,24 +144,29 @@ export class MemoryStorage extends BaseStorage {
|
|
|
150
144
|
// Create cursor for next page if there are more results
|
|
151
145
|
const nextCursor = hasMore ? `${offset + limit}` : undefined;
|
|
152
146
|
// Fetch the actual nouns for the current page
|
|
147
|
+
// v4.0.0: Return HNSWNounWithMetadata (includes metadata field)
|
|
153
148
|
const items = [];
|
|
154
149
|
for (const id of paginatedIds) {
|
|
155
150
|
const noun = this.nouns.get(id);
|
|
156
151
|
if (!noun)
|
|
157
152
|
continue;
|
|
158
|
-
//
|
|
159
|
-
const
|
|
153
|
+
// Get metadata from separate storage
|
|
154
|
+
const metadata = await this.getNounMetadata(id);
|
|
155
|
+
if (!metadata)
|
|
156
|
+
continue; // Skip if no metadata
|
|
157
|
+
// v4.0.0: Create HNSWNounWithMetadata with metadata field
|
|
158
|
+
const nounWithMetadata = {
|
|
160
159
|
id: noun.id,
|
|
161
160
|
vector: [...noun.vector],
|
|
162
161
|
connections: new Map(),
|
|
163
162
|
level: noun.level || 0,
|
|
164
|
-
metadata:
|
|
163
|
+
metadata: metadata // Include metadata field
|
|
165
164
|
};
|
|
166
165
|
// Copy connections
|
|
167
166
|
for (const [level, connections] of noun.connections.entries()) {
|
|
168
|
-
|
|
167
|
+
nounWithMetadata.connections.set(level, new Set(connections));
|
|
169
168
|
}
|
|
170
|
-
items.push(
|
|
169
|
+
items.push(nounWithMetadata);
|
|
171
170
|
}
|
|
172
171
|
return {
|
|
173
172
|
items,
|
|
@@ -178,6 +177,7 @@ export class MemoryStorage extends BaseStorage {
|
|
|
178
177
|
}
|
|
179
178
|
/**
|
|
180
179
|
* Get nouns with pagination - simplified interface for compatibility
|
|
180
|
+
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
|
|
181
181
|
*/
|
|
182
182
|
async getNounsWithPagination(options = {}) {
|
|
183
183
|
// Convert to the getNouns format
|
|
@@ -210,33 +210,33 @@ export class MemoryStorage extends BaseStorage {
|
|
|
210
210
|
return result.items;
|
|
211
211
|
}
|
|
212
212
|
/**
|
|
213
|
-
* Delete a noun from storage
|
|
213
|
+
* Delete a noun from storage (v4.0.0)
|
|
214
214
|
*/
|
|
215
215
|
async deleteNoun_internal(id) {
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
216
|
+
// v4.0.0: Get type from separate metadata storage
|
|
217
|
+
const metadata = await this.getNounMetadata(id);
|
|
218
|
+
if (metadata) {
|
|
219
|
+
const type = metadata.noun || 'default';
|
|
219
220
|
this.decrementEntityCount(type);
|
|
220
221
|
}
|
|
221
222
|
this.nouns.delete(id);
|
|
222
223
|
}
|
|
223
224
|
/**
|
|
224
|
-
* Save a verb to storage
|
|
225
|
+
* Save a verb to storage (v4.0.0: pure vector + core fields, no metadata)
|
|
225
226
|
*/
|
|
226
227
|
async saveVerb_internal(verb) {
|
|
227
228
|
const isNew = !this.verbs.has(verb.id);
|
|
228
229
|
// Create a deep copy to avoid reference issues
|
|
229
|
-
//
|
|
230
|
+
// v4.0.0: Include core relational fields but NO metadata field
|
|
230
231
|
const verbCopy = {
|
|
231
232
|
id: verb.id,
|
|
232
233
|
vector: [...verb.vector],
|
|
233
234
|
connections: new Map(),
|
|
234
|
-
// CORE RELATIONAL DATA
|
|
235
|
+
// CORE RELATIONAL DATA (part of HNSWVerb in v4.0.0)
|
|
235
236
|
verb: verb.verb,
|
|
236
237
|
sourceId: verb.sourceId,
|
|
237
|
-
targetId: verb.targetId
|
|
238
|
-
//
|
|
239
|
-
metadata: verb.metadata
|
|
238
|
+
targetId: verb.targetId
|
|
239
|
+
// ✅ NO metadata field in v4.0.0
|
|
240
240
|
};
|
|
241
241
|
// Copy connections
|
|
242
242
|
for (const [level, connections] of verb.connections.entries()) {
|
|
@@ -244,12 +244,11 @@ export class MemoryStorage extends BaseStorage {
|
|
|
244
244
|
}
|
|
245
245
|
// Save the verb directly in the verbs map
|
|
246
246
|
this.verbs.set(verb.id, verbCopy);
|
|
247
|
-
// Count tracking
|
|
248
|
-
// since HNSWVerb doesn't contain type information
|
|
247
|
+
// Note: Count tracking happens in saveVerbMetadata since metadata is separate
|
|
249
248
|
}
|
|
250
249
|
/**
|
|
251
|
-
* Get a verb from storage (
|
|
252
|
-
*
|
|
250
|
+
* Get a verb from storage (v4.0.0: returns pure vector + core fields)
|
|
251
|
+
* Base class handles combining with metadata
|
|
253
252
|
*/
|
|
254
253
|
async getVerb_internal(id) {
|
|
255
254
|
// Get the verb directly from the verbs map
|
|
@@ -259,17 +258,16 @@ export class MemoryStorage extends BaseStorage {
|
|
|
259
258
|
return null;
|
|
260
259
|
}
|
|
261
260
|
// Return a deep copy of the HNSWVerb
|
|
262
|
-
//
|
|
261
|
+
// v4.0.0: Include core relational fields but NO metadata field
|
|
263
262
|
const verbCopy = {
|
|
264
263
|
id: verb.id,
|
|
265
264
|
vector: [...verb.vector],
|
|
266
265
|
connections: new Map(),
|
|
267
|
-
// CORE RELATIONAL DATA
|
|
266
|
+
// CORE RELATIONAL DATA (part of HNSWVerb in v4.0.0)
|
|
268
267
|
verb: verb.verb,
|
|
269
268
|
sourceId: verb.sourceId,
|
|
270
|
-
targetId: verb.targetId
|
|
271
|
-
//
|
|
272
|
-
metadata: verb.metadata
|
|
269
|
+
targetId: verb.targetId
|
|
270
|
+
// ✅ NO metadata field in v4.0.0
|
|
273
271
|
};
|
|
274
272
|
// Copy connections
|
|
275
273
|
for (const [level, connections] of verb.connections.entries()) {
|
|
@@ -279,8 +277,9 @@ export class MemoryStorage extends BaseStorage {
|
|
|
279
277
|
}
|
|
280
278
|
/**
|
|
281
279
|
* Get verbs with pagination and filtering
|
|
280
|
+
* v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field)
|
|
282
281
|
* @param options Pagination and filtering options
|
|
283
|
-
* @returns Promise that resolves to a paginated result of verbs
|
|
282
|
+
* @returns Promise that resolves to a paginated result of verbs with metadata
|
|
284
283
|
*/
|
|
285
284
|
async getVerbs(options = {}) {
|
|
286
285
|
const pagination = options.pagination || {};
|
|
@@ -304,26 +303,31 @@ export class MemoryStorage extends BaseStorage {
|
|
|
304
303
|
// First, collect all verb IDs that match the filter criteria
|
|
305
304
|
const matchingIds = [];
|
|
306
305
|
// Iterate through all verbs to find matches
|
|
306
|
+
// v4.0.0: Core fields (verb, sourceId, targetId) are in HNSWVerb, not metadata
|
|
307
307
|
for (const [verbId, hnswVerb] of this.verbs.entries()) {
|
|
308
|
-
// Get the metadata for
|
|
308
|
+
// Get the metadata for service/data filtering
|
|
309
309
|
const metadata = await this.getVerbMetadata(verbId);
|
|
310
310
|
// Filter by verb type if specified
|
|
311
|
-
|
|
311
|
+
// v4.0.0: verb type is in HNSWVerb.verb
|
|
312
|
+
if (verbTypes && !verbTypes.includes(hnswVerb.verb || '')) {
|
|
312
313
|
continue;
|
|
313
314
|
}
|
|
314
315
|
// Filter by source ID if specified
|
|
315
|
-
|
|
316
|
+
// v4.0.0: sourceId is in HNSWVerb.sourceId
|
|
317
|
+
if (sourceIds && !sourceIds.includes(hnswVerb.sourceId || '')) {
|
|
316
318
|
continue;
|
|
317
319
|
}
|
|
318
320
|
// Filter by target ID if specified
|
|
319
|
-
|
|
321
|
+
// v4.0.0: targetId is in HNSWVerb.targetId
|
|
322
|
+
if (targetIds && !targetIds.includes(hnswVerb.targetId || '')) {
|
|
320
323
|
continue;
|
|
321
324
|
}
|
|
322
325
|
// Filter by metadata fields if specified
|
|
323
|
-
if (filter.metadata && metadata
|
|
326
|
+
if (filter.metadata && metadata) {
|
|
324
327
|
let metadataMatch = true;
|
|
325
328
|
for (const [key, value] of Object.entries(filter.metadata)) {
|
|
326
|
-
|
|
329
|
+
const metadataValue = metadata[key];
|
|
330
|
+
if (metadataValue !== value) {
|
|
327
331
|
metadataMatch = false;
|
|
328
332
|
break;
|
|
329
333
|
}
|
|
@@ -332,8 +336,7 @@ export class MemoryStorage extends BaseStorage {
|
|
|
332
336
|
continue;
|
|
333
337
|
}
|
|
334
338
|
// Filter by service if specified
|
|
335
|
-
if (services && metadata && metadata.
|
|
336
|
-
!services.includes(metadata.createdBy.augmentation)) {
|
|
339
|
+
if (services && metadata && metadata.service && !services.includes(metadata.service)) {
|
|
337
340
|
continue;
|
|
338
341
|
}
|
|
339
342
|
// If we got here, the verb matches all filters
|
|
@@ -346,41 +349,33 @@ export class MemoryStorage extends BaseStorage {
|
|
|
346
349
|
// Create cursor for next page if there are more results
|
|
347
350
|
const nextCursor = hasMore ? `${offset + limit}` : undefined;
|
|
348
351
|
// Fetch the actual verbs for the current page
|
|
352
|
+
// v4.0.0: Return HNSWVerbWithMetadata (includes metadata field)
|
|
349
353
|
const items = [];
|
|
350
354
|
for (const id of paginatedIds) {
|
|
351
355
|
const hnswVerb = this.verbs.get(id);
|
|
352
|
-
const metadata = await this.getVerbMetadata(id);
|
|
353
356
|
if (!hnswVerb)
|
|
354
357
|
continue;
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
sourceId: '',
|
|
362
|
-
targetId: ''
|
|
363
|
-
});
|
|
364
|
-
continue;
|
|
365
|
-
}
|
|
366
|
-
// Create a complete GraphVerb by combining HNSWVerb with metadata
|
|
367
|
-
const graphVerb = {
|
|
358
|
+
// Get metadata from separate storage
|
|
359
|
+
const metadata = await this.getVerbMetadata(id);
|
|
360
|
+
if (!metadata)
|
|
361
|
+
continue; // Skip if no metadata
|
|
362
|
+
// v4.0.0: Create HNSWVerbWithMetadata with metadata field
|
|
363
|
+
const verbWithMetadata = {
|
|
368
364
|
id: hnswVerb.id,
|
|
369
365
|
vector: [...hnswVerb.vector],
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
createdAt: metadata.createdAt,
|
|
378
|
-
updatedAt: metadata.updatedAt,
|
|
379
|
-
createdBy: metadata.createdBy,
|
|
380
|
-
data: metadata.data,
|
|
381
|
-
metadata: metadata.metadata || metadata.data // Use metadata.metadata (user's custom metadata)
|
|
366
|
+
connections: new Map(),
|
|
367
|
+
// Core relational fields (part of HNSWVerb)
|
|
368
|
+
verb: hnswVerb.verb,
|
|
369
|
+
sourceId: hnswVerb.sourceId,
|
|
370
|
+
targetId: hnswVerb.targetId,
|
|
371
|
+
// Metadata field
|
|
372
|
+
metadata: metadata
|
|
382
373
|
};
|
|
383
|
-
|
|
374
|
+
// Copy connections
|
|
375
|
+
for (const [level, connections] of hnswVerb.connections.entries()) {
|
|
376
|
+
verbWithMetadata.connections.set(level, new Set(connections));
|
|
377
|
+
}
|
|
378
|
+
items.push(verbWithMetadata);
|
|
384
379
|
}
|
|
385
380
|
return {
|
|
386
381
|
items,
|
|
@@ -601,22 +596,32 @@ export class MemoryStorage extends BaseStorage {
|
|
|
601
596
|
// to check multiple storage locations
|
|
602
597
|
}
|
|
603
598
|
/**
|
|
604
|
-
* Initialize counts from in-memory storage - O(1) operation
|
|
599
|
+
* Initialize counts from in-memory storage - O(1) operation (v4.0.0)
|
|
605
600
|
*/
|
|
606
601
|
async initializeCounts() {
|
|
607
602
|
// For memory storage, initialize counts from current in-memory state
|
|
608
603
|
this.totalNounCount = this.nouns.size;
|
|
609
|
-
this.totalVerbCount = this.
|
|
610
|
-
// Initialize type-based counts by scanning
|
|
604
|
+
this.totalVerbCount = this.verbs.size;
|
|
605
|
+
// Initialize type-based counts by scanning metadata storage (v4.0.0)
|
|
611
606
|
this.entityCounts.clear();
|
|
612
607
|
this.verbCounts.clear();
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
608
|
+
// Count nouns by loading metadata for each
|
|
609
|
+
for (const [nounId, noun] of this.nouns.entries()) {
|
|
610
|
+
const metadata = await this.getNounMetadata(nounId);
|
|
611
|
+
if (metadata) {
|
|
612
|
+
const type = metadata.noun || 'default';
|
|
613
|
+
this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1);
|
|
614
|
+
}
|
|
616
615
|
}
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
616
|
+
// Count verbs by loading metadata for each
|
|
617
|
+
for (const [verbId, verb] of this.verbs.entries()) {
|
|
618
|
+
const metadata = await this.getVerbMetadata(verbId);
|
|
619
|
+
if (metadata) {
|
|
620
|
+
// VerbMetadata doesn't have verb type - that's in HNSWVerb now
|
|
621
|
+
// Use the verb's type from the HNSWVerb itself
|
|
622
|
+
const type = verb.verb || 'default';
|
|
623
|
+
this.verbCounts.set(type, (this.verbCounts.get(type) || 0) + 1);
|
|
624
|
+
}
|
|
620
625
|
}
|
|
621
626
|
}
|
|
622
627
|
/**
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* OPFS (Origin Private File System) Storage Adapter
|
|
3
3
|
* Provides persistent storage for the vector database using the Origin Private File System API
|
|
4
4
|
*/
|
|
5
|
-
import {
|
|
5
|
+
import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
|
|
6
6
|
import { BaseStorage } from '../baseStorage.js';
|
|
7
7
|
import '../../types/fileSystemTypes.js';
|
|
8
8
|
type HNSWNode = HNSWNoun;
|
|
@@ -87,7 +87,8 @@ export declare class OPFSStorage extends BaseStorage {
|
|
|
87
87
|
protected saveEdge(edge: Edge): Promise<void>;
|
|
88
88
|
/**
|
|
89
89
|
* Get a verb from storage (internal implementation)
|
|
90
|
-
*
|
|
90
|
+
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
91
|
+
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
91
92
|
*/
|
|
92
93
|
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
|
93
94
|
/**
|
|
@@ -101,27 +102,27 @@ export declare class OPFSStorage extends BaseStorage {
|
|
|
101
102
|
/**
|
|
102
103
|
* Get verbs by source (internal implementation)
|
|
103
104
|
*/
|
|
104
|
-
protected getVerbsBySource_internal(sourceId: string): Promise<
|
|
105
|
+
protected getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
105
106
|
/**
|
|
106
107
|
* Get edges by source
|
|
107
108
|
*/
|
|
108
|
-
protected getEdgesBySource(sourceId: string): Promise<
|
|
109
|
+
protected getEdgesBySource(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
109
110
|
/**
|
|
110
111
|
* Get verbs by target (internal implementation)
|
|
111
112
|
*/
|
|
112
|
-
protected getVerbsByTarget_internal(targetId: string): Promise<
|
|
113
|
+
protected getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
113
114
|
/**
|
|
114
115
|
* Get edges by target
|
|
115
116
|
*/
|
|
116
|
-
protected getEdgesByTarget(targetId: string): Promise<
|
|
117
|
+
protected getEdgesByTarget(targetId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
117
118
|
/**
|
|
118
119
|
* Get verbs by type (internal implementation)
|
|
119
120
|
*/
|
|
120
|
-
protected getVerbsByType_internal(type: string): Promise<
|
|
121
|
+
protected getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]>;
|
|
121
122
|
/**
|
|
122
123
|
* Get edges by type
|
|
123
124
|
*/
|
|
124
|
-
protected getEdgesByType(type: string): Promise<
|
|
125
|
+
protected getEdgesByType(type: string): Promise<HNSWVerbWithMetadata[]>;
|
|
125
126
|
/**
|
|
126
127
|
* Delete a verb from storage (internal implementation)
|
|
127
128
|
*/
|
|
@@ -159,6 +160,10 @@ export declare class OPFSStorage extends BaseStorage {
|
|
|
159
160
|
* Clear all data from storage
|
|
160
161
|
*/
|
|
161
162
|
clear(): Promise<void>;
|
|
163
|
+
private quotaWarningThreshold;
|
|
164
|
+
private quotaCriticalThreshold;
|
|
165
|
+
private lastQuotaCheck;
|
|
166
|
+
private quotaCheckInterval;
|
|
162
167
|
/**
|
|
163
168
|
* Get information about storage usage and capacity
|
|
164
169
|
*/
|
|
@@ -168,6 +173,39 @@ export declare class OPFSStorage extends BaseStorage {
|
|
|
168
173
|
quota: number | null;
|
|
169
174
|
details?: Record<string, any>;
|
|
170
175
|
}>;
|
|
176
|
+
/**
|
|
177
|
+
* Get detailed quota status with warnings (v4.0.0)
|
|
178
|
+
* Monitors storage usage and warns when approaching quota limits
|
|
179
|
+
*
|
|
180
|
+
* @returns Promise that resolves to quota status with warning levels
|
|
181
|
+
*
|
|
182
|
+
* @example
|
|
183
|
+
* const status = await storage.getQuotaStatus()
|
|
184
|
+
* if (status.warning) {
|
|
185
|
+
* console.warn(`Storage ${status.usagePercent}% full: ${status.warningMessage}`)
|
|
186
|
+
* }
|
|
187
|
+
*/
|
|
188
|
+
getQuotaStatus(): Promise<{
|
|
189
|
+
usage: number;
|
|
190
|
+
quota: number | null;
|
|
191
|
+
usagePercent: number;
|
|
192
|
+
remaining: number | null;
|
|
193
|
+
status: 'ok' | 'warning' | 'critical';
|
|
194
|
+
warning: boolean;
|
|
195
|
+
warningMessage?: string;
|
|
196
|
+
}>;
|
|
197
|
+
/**
|
|
198
|
+
* Monitor quota during operations (v4.0.0)
|
|
199
|
+
* Automatically checks quota at regular intervals and warns if approaching limits
|
|
200
|
+
* Call this before write operations to ensure quota is available
|
|
201
|
+
*
|
|
202
|
+
* @returns Promise that resolves when quota check is complete
|
|
203
|
+
*
|
|
204
|
+
* @example
|
|
205
|
+
* await storage.monitorQuota() // Checks quota if interval has passed
|
|
206
|
+
* await storage.saveNoun(noun) // Proceed with write operation
|
|
207
|
+
*/
|
|
208
|
+
monitorQuota(): Promise<void>;
|
|
171
209
|
/**
|
|
172
210
|
* Get the statistics key for a specific date
|
|
173
211
|
* @param date The date to get the key for
|
|
@@ -226,7 +264,7 @@ export declare class OPFSStorage extends BaseStorage {
|
|
|
226
264
|
metadata?: Record<string, any>;
|
|
227
265
|
};
|
|
228
266
|
}): Promise<{
|
|
229
|
-
items:
|
|
267
|
+
items: HNSWNounWithMetadata[];
|
|
230
268
|
totalCount?: number;
|
|
231
269
|
hasMore: boolean;
|
|
232
270
|
nextCursor?: string;
|
|
@@ -247,7 +285,7 @@ export declare class OPFSStorage extends BaseStorage {
|
|
|
247
285
|
metadata?: Record<string, any>;
|
|
248
286
|
};
|
|
249
287
|
}): Promise<{
|
|
250
|
-
items:
|
|
288
|
+
items: HNSWVerbWithMetadata[];
|
|
251
289
|
totalCount?: number;
|
|
252
290
|
hasMore: boolean;
|
|
253
291
|
nextCursor?: string;
|