@soulcraft/brainy 3.50.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +201 -0
- package/README.md +358 -658
- package/dist/api/ConfigAPI.js +56 -19
- package/dist/api/DataAPI.js +24 -18
- package/dist/augmentations/storageAugmentations.d.ts +24 -0
- package/dist/augmentations/storageAugmentations.js +22 -0
- package/dist/brainy.js +32 -9
- package/dist/cli/commands/core.d.ts +20 -10
- package/dist/cli/commands/core.js +384 -82
- package/dist/cli/commands/import.d.ts +41 -0
- package/dist/cli/commands/import.js +456 -0
- package/dist/cli/commands/insights.d.ts +34 -0
- package/dist/cli/commands/insights.js +300 -0
- package/dist/cli/commands/neural.d.ts +6 -12
- package/dist/cli/commands/neural.js +113 -10
- package/dist/cli/commands/nlp.d.ts +28 -0
- package/dist/cli/commands/nlp.js +246 -0
- package/dist/cli/commands/storage.d.ts +64 -0
- package/dist/cli/commands/storage.js +730 -0
- package/dist/cli/index.js +210 -24
- package/dist/coreTypes.d.ts +206 -34
- package/dist/distributed/configManager.js +8 -6
- package/dist/distributed/shardMigration.js +2 -0
- package/dist/distributed/storageDiscovery.js +6 -4
- package/dist/embeddings/EmbeddingManager.d.ts +2 -2
- package/dist/embeddings/EmbeddingManager.js +5 -1
- package/dist/graph/lsm/LSMTree.js +32 -20
- package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
- package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
- package/dist/storage/adapters/azureBlobStorage.js +1809 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
- package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
- package/dist/storage/adapters/fileSystemStorage.js +204 -127
- package/dist/storage/adapters/gcsStorage.d.ts +119 -9
- package/dist/storage/adapters/gcsStorage.js +317 -62
- package/dist/storage/adapters/memoryStorage.d.ts +30 -18
- package/dist/storage/adapters/memoryStorage.js +99 -94
- package/dist/storage/adapters/opfsStorage.d.ts +48 -10
- package/dist/storage/adapters/opfsStorage.js +201 -80
- package/dist/storage/adapters/r2Storage.d.ts +12 -5
- package/dist/storage/adapters/r2Storage.js +63 -15
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
- package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
- package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
- package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
- package/dist/storage/baseStorage.d.ts +41 -38
- package/dist/storage/baseStorage.js +110 -134
- package/dist/storage/storageFactory.d.ts +29 -2
- package/dist/storage/storageFactory.js +30 -1
- package/dist/utils/entityIdMapper.js +5 -2
- package/dist/utils/fieldTypeInference.js +8 -1
- package/dist/utils/metadataFilter.d.ts +3 -2
- package/dist/utils/metadataFilter.js +1 -0
- package/dist/utils/metadataIndex.js +2 -0
- package/dist/utils/metadataIndexChunking.js +9 -4
- package/dist/utils/periodicCleanup.js +1 -0
- package/package.json +3 -1
package/dist/coreTypes.d.ts
CHANGED
|
@@ -46,8 +46,9 @@ export interface PaginatedSearchResult<T = any> {
|
|
|
46
46
|
export type DistanceFunction = (a: Vector, b: Vector) => number;
|
|
47
47
|
/**
|
|
48
48
|
* Embedding function for converting data to vectors
|
|
49
|
+
* v4.0.0: Now properly typed - accepts string, string array (batch), or object, no `any`
|
|
49
50
|
*/
|
|
50
|
-
export type EmbeddingFunction = (data:
|
|
51
|
+
export type EmbeddingFunction = (data: string | string[] | Record<string, unknown>) => Promise<Vector>;
|
|
51
52
|
/**
|
|
52
53
|
* Embedding model interface
|
|
53
54
|
*/
|
|
@@ -58,38 +59,49 @@ export interface EmbeddingModel {
|
|
|
58
59
|
init(): Promise<void>;
|
|
59
60
|
/**
|
|
60
61
|
* Embed data into a vector
|
|
62
|
+
* v4.0.0: Now properly typed - accepts string, string array (batch), or object, no `any`
|
|
61
63
|
*/
|
|
62
|
-
embed(data:
|
|
64
|
+
embed(data: string | string[] | Record<string, unknown>): Promise<Vector>;
|
|
63
65
|
/**
|
|
64
66
|
* Dispose of the model resources
|
|
65
67
|
*/
|
|
66
68
|
dispose(): Promise<void>;
|
|
67
69
|
}
|
|
68
70
|
/**
|
|
69
|
-
* HNSW graph noun
|
|
71
|
+
* HNSW graph noun - Pure vector structure (v4.0.0)
|
|
72
|
+
*
|
|
73
|
+
* v4.0.0 BREAKING CHANGE: metadata field removed
|
|
74
|
+
* - Stores ONLY vector data for optimal memory usage
|
|
75
|
+
* - Metadata stored separately and combined on retrieval
|
|
76
|
+
* - 25% memory reduction @ 1B scale (no in-memory metadata)
|
|
77
|
+
* - Prevents metadata explosion bugs at compile-time
|
|
70
78
|
*/
|
|
71
79
|
export interface HNSWNoun {
|
|
72
80
|
id: string;
|
|
73
81
|
vector: Vector;
|
|
74
82
|
connections: Map<number, Set<string>>;
|
|
75
83
|
level: number;
|
|
76
|
-
metadata?: any;
|
|
77
84
|
}
|
|
78
85
|
/**
|
|
79
|
-
* Lightweight verb for HNSW index storage
|
|
80
|
-
* Contains essential data including core relational fields
|
|
86
|
+
* Lightweight verb for HNSW index storage - Core relational structure (v4.0.0)
|
|
81
87
|
*
|
|
82
|
-
*
|
|
88
|
+
* Core fields (v3.50.1+): verb/sourceId/targetId are first-class fields
|
|
83
89
|
* These are NOT metadata - they're the essence of what a verb IS:
|
|
84
90
|
* - verb: The relationship type (creates, contains, etc.) - needed for routing & display
|
|
85
91
|
* - sourceId: What entity this verb connects FROM - needed for graph traversal
|
|
86
92
|
* - targetId: What entity this verb connects TO - needed for graph traversal
|
|
87
93
|
*
|
|
94
|
+
* v4.0.0 BREAKING CHANGE: metadata field removed
|
|
95
|
+
* - Stores ONLY vector + core relational data
|
|
96
|
+
* - User metadata (weight, custom fields) stored separately
|
|
97
|
+
* - 10x faster metadata-only updates (skip HNSW rebuild)
|
|
98
|
+
* - Prevents metadata explosion bugs at compile-time
|
|
99
|
+
*
|
|
88
100
|
* Benefits:
|
|
89
|
-
* - ONE file read
|
|
101
|
+
* - ONE file read for graph operations (core fields always available)
|
|
90
102
|
* - No type caching needed (type is always available)
|
|
91
103
|
* - Faster graph traversal (source/target immediately available)
|
|
92
|
-
* -
|
|
104
|
+
* - Optimal memory usage (no user metadata in HNSW)
|
|
93
105
|
*/
|
|
94
106
|
export interface HNSWVerb {
|
|
95
107
|
id: string;
|
|
@@ -98,11 +110,88 @@ export interface HNSWVerb {
|
|
|
98
110
|
verb: VerbType;
|
|
99
111
|
sourceId: string;
|
|
100
112
|
targetId: string;
|
|
101
|
-
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Noun metadata structure (v4.0.0)
|
|
116
|
+
*
|
|
117
|
+
* Stores all metadata separately from vector data.
|
|
118
|
+
* Combines with HNSWNoun to form complete entity.
|
|
119
|
+
*/
|
|
120
|
+
export interface NounMetadata {
|
|
121
|
+
noun: string;
|
|
122
|
+
data?: unknown;
|
|
123
|
+
createdAt?: {
|
|
124
|
+
seconds: number;
|
|
125
|
+
nanoseconds: number;
|
|
126
|
+
} | number;
|
|
127
|
+
updatedAt?: {
|
|
128
|
+
seconds: number;
|
|
129
|
+
nanoseconds: number;
|
|
130
|
+
} | number;
|
|
131
|
+
createdBy?: {
|
|
132
|
+
augmentation: string;
|
|
133
|
+
version: string;
|
|
134
|
+
};
|
|
135
|
+
service?: string;
|
|
136
|
+
[key: string]: unknown;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Verb metadata structure (v4.0.0)
|
|
140
|
+
*
|
|
141
|
+
* Stores all metadata separately from vector + core relational data.
|
|
142
|
+
* Core fields (verb, sourceId, targetId) remain in HNSWVerb.
|
|
143
|
+
*/
|
|
144
|
+
export interface VerbMetadata {
|
|
145
|
+
weight?: number;
|
|
146
|
+
data?: unknown;
|
|
147
|
+
createdAt?: {
|
|
148
|
+
seconds: number;
|
|
149
|
+
nanoseconds: number;
|
|
150
|
+
} | number;
|
|
151
|
+
updatedAt?: {
|
|
152
|
+
seconds: number;
|
|
153
|
+
nanoseconds: number;
|
|
154
|
+
} | number;
|
|
155
|
+
createdBy?: {
|
|
156
|
+
augmentation: string;
|
|
157
|
+
version: string;
|
|
158
|
+
};
|
|
159
|
+
service?: string;
|
|
160
|
+
[key: string]: unknown;
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Combined noun structure for transport/API boundaries (v4.0.0)
|
|
164
|
+
*
|
|
165
|
+
* Combines pure HNSWNoun vector + separate NounMetadata.
|
|
166
|
+
* Used for API responses and storage retrieval.
|
|
167
|
+
*/
|
|
168
|
+
export interface HNSWNounWithMetadata {
|
|
169
|
+
id: string;
|
|
170
|
+
vector: Vector;
|
|
171
|
+
connections: Map<number, Set<string>>;
|
|
172
|
+
level: number;
|
|
173
|
+
metadata: NounMetadata;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Combined verb structure for transport/API boundaries (v4.0.0)
|
|
177
|
+
*
|
|
178
|
+
* Combines pure HNSWVerb (vector + core fields) + separate VerbMetadata.
|
|
179
|
+
* Used for API responses and storage retrieval.
|
|
180
|
+
*/
|
|
181
|
+
export interface HNSWVerbWithMetadata {
|
|
182
|
+
id: string;
|
|
183
|
+
vector: Vector;
|
|
184
|
+
connections: Map<number, Set<string>>;
|
|
185
|
+
verb: VerbType;
|
|
186
|
+
sourceId: string;
|
|
187
|
+
targetId: string;
|
|
188
|
+
metadata: VerbMetadata;
|
|
102
189
|
}
|
|
103
190
|
/**
|
|
104
191
|
* Verb representing a relationship between nouns
|
|
105
192
|
* Stored separately from HNSW index for lightweight performance
|
|
193
|
+
*
|
|
194
|
+
* @deprecated Will be replaced by HNSWVerbWithMetadata in future versions
|
|
106
195
|
*/
|
|
107
196
|
export interface GraphVerb {
|
|
108
197
|
id: string;
|
|
@@ -346,10 +435,41 @@ export interface StatisticsData {
|
|
|
346
435
|
*/
|
|
347
436
|
distributedConfig?: import('./types/distributedTypes.js').SharedConfig;
|
|
348
437
|
}
|
|
438
|
+
/**
|
|
439
|
+
* Change record for getChangesSince (v4.0.0)
|
|
440
|
+
* Replaces `any[]` with properly typed structure
|
|
441
|
+
*/
|
|
442
|
+
export interface Change {
|
|
443
|
+
id: string;
|
|
444
|
+
type: 'noun' | 'verb';
|
|
445
|
+
operation: 'create' | 'update' | 'delete';
|
|
446
|
+
timestamp: number;
|
|
447
|
+
data?: HNSWNounWithMetadata | HNSWVerbWithMetadata;
|
|
448
|
+
}
|
|
349
449
|
export interface StorageAdapter {
|
|
350
450
|
init(): Promise<void>;
|
|
451
|
+
/**
|
|
452
|
+
* Save noun - Pure HNSW vector data only (v4.0.0)
|
|
453
|
+
* @param noun Pure HNSW vector data (no metadata)
|
|
454
|
+
* Note: Use saveNounMetadata() to save metadata separately
|
|
455
|
+
*/
|
|
351
456
|
saveNoun(noun: HNSWNoun): Promise<void>;
|
|
352
|
-
|
|
457
|
+
/**
|
|
458
|
+
* Save noun metadata separately (v4.0.0)
|
|
459
|
+
* @param id Noun ID
|
|
460
|
+
* @param metadata Noun metadata
|
|
461
|
+
*/
|
|
462
|
+
saveNounMetadata(id: string, metadata: NounMetadata): Promise<void>;
|
|
463
|
+
/**
|
|
464
|
+
* Delete noun metadata (v4.0.0)
|
|
465
|
+
* @param id Noun ID
|
|
466
|
+
*/
|
|
467
|
+
deleteNounMetadata(id: string): Promise<void>;
|
|
468
|
+
/**
|
|
469
|
+
* Get noun with metadata combined (v4.0.0)
|
|
470
|
+
* @returns Combined HNSWNounWithMetadata or null
|
|
471
|
+
*/
|
|
472
|
+
getNoun(id: string): Promise<HNSWNounWithMetadata | null>;
|
|
353
473
|
/**
|
|
354
474
|
* Get nouns with pagination and filtering
|
|
355
475
|
* @param options Pagination and filtering options
|
|
@@ -367,7 +487,7 @@ export interface StorageAdapter {
|
|
|
367
487
|
metadata?: Record<string, any>;
|
|
368
488
|
};
|
|
369
489
|
}): Promise<{
|
|
370
|
-
items:
|
|
490
|
+
items: HNSWNounWithMetadata[];
|
|
371
491
|
totalCount?: number;
|
|
372
492
|
hasMore: boolean;
|
|
373
493
|
nextCursor?: string;
|
|
@@ -378,10 +498,19 @@ export interface StorageAdapter {
|
|
|
378
498
|
* @returns Promise that resolves to an array of nouns of the specified noun type
|
|
379
499
|
* @deprecated Use getNouns() with filter.nounType instead
|
|
380
500
|
*/
|
|
381
|
-
getNounsByNounType(nounType: string): Promise<
|
|
501
|
+
getNounsByNounType(nounType: string): Promise<HNSWNounWithMetadata[]>;
|
|
382
502
|
deleteNoun(id: string): Promise<void>;
|
|
383
|
-
|
|
384
|
-
|
|
503
|
+
/**
|
|
504
|
+
* Save verb - Pure HNSW verb with core fields only (v4.0.0)
|
|
505
|
+
* @param verb Pure HNSW verb data (vector + core fields, no user metadata)
|
|
506
|
+
* Note: Use saveVerbMetadata() to save metadata separately
|
|
507
|
+
*/
|
|
508
|
+
saveVerb(verb: HNSWVerb): Promise<void>;
|
|
509
|
+
/**
|
|
510
|
+
* Get verb with metadata combined (v4.0.0)
|
|
511
|
+
* @returns Combined HNSWVerbWithMetadata or null
|
|
512
|
+
*/
|
|
513
|
+
getVerb(id: string): Promise<HNSWVerbWithMetadata | null>;
|
|
385
514
|
/**
|
|
386
515
|
* Get verbs with pagination and filtering
|
|
387
516
|
* @param options Pagination and filtering options
|
|
@@ -401,7 +530,7 @@ export interface StorageAdapter {
|
|
|
401
530
|
metadata?: Record<string, any>;
|
|
402
531
|
};
|
|
403
532
|
}): Promise<{
|
|
404
|
-
items:
|
|
533
|
+
items: HNSWVerbWithMetadata[];
|
|
405
534
|
totalCount?: number;
|
|
406
535
|
hasMore: boolean;
|
|
407
536
|
nextCursor?: string;
|
|
@@ -412,50 +541,93 @@ export interface StorageAdapter {
|
|
|
412
541
|
* @returns Promise that resolves to an array of verbs with the specified source ID
|
|
413
542
|
* @deprecated Use getVerbs() with filter.sourceId instead
|
|
414
543
|
*/
|
|
415
|
-
getVerbsBySource(sourceId: string): Promise<
|
|
544
|
+
getVerbsBySource(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
416
545
|
/**
|
|
417
546
|
* Get verbs by target
|
|
418
547
|
* @param targetId The target ID to filter by
|
|
419
548
|
* @returns Promise that resolves to an array of verbs with the specified target ID
|
|
420
549
|
* @deprecated Use getVerbs() with filter.targetId instead
|
|
421
550
|
*/
|
|
422
|
-
getVerbsByTarget(targetId: string): Promise<
|
|
551
|
+
getVerbsByTarget(targetId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
423
552
|
/**
|
|
424
553
|
* Get verbs by type
|
|
425
554
|
* @param type The verb type to filter by
|
|
426
555
|
* @returns Promise that resolves to an array of verbs with the specified type
|
|
427
556
|
* @deprecated Use getVerbs() with filter.verbType instead
|
|
428
557
|
*/
|
|
429
|
-
getVerbsByType(type: string): Promise<
|
|
558
|
+
getVerbsByType(type: string): Promise<HNSWVerbWithMetadata[]>;
|
|
430
559
|
deleteVerb(id: string): Promise<void>;
|
|
431
|
-
|
|
432
|
-
|
|
560
|
+
/**
|
|
561
|
+
* Save metadata (v4.0.0: now typed)
|
|
562
|
+
* @param id Entity ID
|
|
563
|
+
* @param metadata Typed noun metadata
|
|
564
|
+
*/
|
|
565
|
+
saveMetadata(id: string, metadata: NounMetadata): Promise<void>;
|
|
566
|
+
/**
|
|
567
|
+
* Get metadata (v4.0.0: now typed)
|
|
568
|
+
* @param id Entity ID
|
|
569
|
+
* @returns Typed noun metadata or null
|
|
570
|
+
*/
|
|
571
|
+
getMetadata(id: string): Promise<NounMetadata | null>;
|
|
433
572
|
/**
|
|
434
573
|
* Get multiple metadata objects in batches (prevents socket exhaustion)
|
|
435
574
|
* @param ids Array of IDs to get metadata for
|
|
436
|
-
* @returns Promise that resolves to a Map of id -> metadata
|
|
575
|
+
* @returns Promise that resolves to a Map of id -> metadata (v4.0.0: typed)
|
|
437
576
|
*/
|
|
438
|
-
getMetadataBatch?(ids: string[]): Promise<Map<string,
|
|
577
|
+
getMetadataBatch?(ids: string[]): Promise<Map<string, NounMetadata>>;
|
|
439
578
|
/**
|
|
440
|
-
* Get noun metadata from storage
|
|
579
|
+
* Get noun metadata from storage (v4.0.0: now typed)
|
|
441
580
|
* @param id The ID of the noun
|
|
442
581
|
* @returns Promise that resolves to the metadata or null if not found
|
|
443
582
|
*/
|
|
444
|
-
getNounMetadata(id: string): Promise<
|
|
583
|
+
getNounMetadata(id: string): Promise<NounMetadata | null>;
|
|
445
584
|
/**
|
|
446
|
-
* Save verb metadata to storage
|
|
585
|
+
* Save verb metadata to storage (v4.0.0: now typed)
|
|
447
586
|
* @param id The ID of the verb
|
|
448
587
|
* @param metadata The metadata to save
|
|
449
588
|
* @returns Promise that resolves when the metadata is saved
|
|
450
589
|
*/
|
|
451
|
-
saveVerbMetadata(id: string, metadata:
|
|
590
|
+
saveVerbMetadata(id: string, metadata: VerbMetadata): Promise<void>;
|
|
452
591
|
/**
|
|
453
|
-
* Get verb metadata from storage
|
|
592
|
+
* Get verb metadata from storage (v4.0.0: now typed)
|
|
454
593
|
* @param id The ID of the verb
|
|
455
594
|
* @returns Promise that resolves to the metadata or null if not found
|
|
456
595
|
*/
|
|
457
|
-
getVerbMetadata(id: string): Promise<
|
|
596
|
+
getVerbMetadata(id: string): Promise<VerbMetadata | null>;
|
|
458
597
|
clear(): Promise<void>;
|
|
598
|
+
/**
|
|
599
|
+
* Batch delete multiple objects from storage (v4.0.0)
|
|
600
|
+
* Efficient deletion of large numbers of entities using cloud provider batch APIs.
|
|
601
|
+
* Significantly faster and cheaper than individual deletes (up to 1000x speedup).
|
|
602
|
+
*
|
|
603
|
+
* @param keys - Array of object keys (paths) to delete
|
|
604
|
+
* @param options - Optional configuration for batch deletion
|
|
605
|
+
* @param options.maxRetries - Maximum number of retry attempts per batch (default: 3)
|
|
606
|
+
* @param options.retryDelayMs - Base delay between retries in milliseconds (default: 1000)
|
|
607
|
+
* @param options.continueOnError - Continue processing remaining batches if one fails (default: true)
|
|
608
|
+
* @returns Promise with deletion statistics
|
|
609
|
+
*
|
|
610
|
+
* @example
|
|
611
|
+
* const result = await storage.batchDelete(
|
|
612
|
+
* ['path1', 'path2', 'path3'],
|
|
613
|
+
* { continueOnError: true }
|
|
614
|
+
* )
|
|
615
|
+
* console.log(`Deleted: ${result.successfulDeletes}/${result.totalRequested}`)
|
|
616
|
+
* console.log(`Failed: ${result.failedDeletes}`)
|
|
617
|
+
*/
|
|
618
|
+
batchDelete?(keys: string[], options?: {
|
|
619
|
+
maxRetries?: number;
|
|
620
|
+
retryDelayMs?: number;
|
|
621
|
+
continueOnError?: boolean;
|
|
622
|
+
}): Promise<{
|
|
623
|
+
totalRequested: number;
|
|
624
|
+
successfulDeletes: number;
|
|
625
|
+
failedDeletes: number;
|
|
626
|
+
errors: Array<{
|
|
627
|
+
key: string;
|
|
628
|
+
error: string;
|
|
629
|
+
}>;
|
|
630
|
+
}>;
|
|
459
631
|
/**
|
|
460
632
|
* Get information about storage usage and capacity
|
|
461
633
|
* @returns Promise that resolves to an object containing storage status information
|
|
@@ -513,11 +685,11 @@ export interface StorageAdapter {
|
|
|
513
685
|
*/
|
|
514
686
|
flushStatisticsToStorage(): Promise<void>;
|
|
515
687
|
/**
|
|
516
|
-
* Track field names from a JSON document
|
|
688
|
+
* Track field names from a JSON document (v4.0.0: now typed)
|
|
517
689
|
* @param jsonDocument The JSON document to extract field names from
|
|
518
690
|
* @param service The service that inserted the data
|
|
519
691
|
*/
|
|
520
|
-
trackFieldNames(jsonDocument:
|
|
692
|
+
trackFieldNames(jsonDocument: Record<string, unknown>, service: string): Promise<void>;
|
|
521
693
|
/**
|
|
522
694
|
* Get available field names by service
|
|
523
695
|
* @returns Record of field names by service
|
|
@@ -529,12 +701,12 @@ export interface StorageAdapter {
|
|
|
529
701
|
*/
|
|
530
702
|
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>;
|
|
531
703
|
/**
|
|
532
|
-
* Get changes since a specific timestamp
|
|
704
|
+
* Get changes since a specific timestamp (v4.0.0: now typed)
|
|
533
705
|
* @param timestamp The timestamp to get changes since
|
|
534
706
|
* @param limit Optional limit on the number of changes to return
|
|
535
|
-
* @returns Promise that resolves to an array of changes
|
|
707
|
+
* @returns Promise that resolves to an array of properly typed changes
|
|
536
708
|
*/
|
|
537
|
-
getChangesSince?(timestamp: number, limit?: number): Promise<
|
|
709
|
+
getChangesSince?(timestamp: number, limit?: number): Promise<Change[]>;
|
|
538
710
|
/**
|
|
539
711
|
* Get total count of nouns in storage - O(1) operation
|
|
540
712
|
* @returns Promise that resolves to the total number of nouns
|
|
@@ -77,9 +77,10 @@ export class DistributedConfigManager {
|
|
|
77
77
|
const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
|
|
78
78
|
if (configData) {
|
|
79
79
|
// Migrate to new location
|
|
80
|
-
|
|
81
|
-
this.
|
|
82
|
-
|
|
80
|
+
const config = configData;
|
|
81
|
+
await this.migrateConfig(config);
|
|
82
|
+
this.lastConfigVersion = config.version;
|
|
83
|
+
return config;
|
|
83
84
|
}
|
|
84
85
|
}
|
|
85
86
|
catch (error) {
|
|
@@ -170,13 +171,14 @@ export class DistributedConfigManager {
|
|
|
170
171
|
const legacyConfig = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
|
|
171
172
|
if (legacyConfig) {
|
|
172
173
|
console.log('Migrating distributed config from legacy location to index folder...');
|
|
174
|
+
const config = legacyConfig;
|
|
173
175
|
// Save to new location
|
|
174
|
-
await this.migrateConfig(
|
|
176
|
+
await this.migrateConfig(config);
|
|
175
177
|
// Delete from old location (optional - we can keep it for rollback)
|
|
176
178
|
// await this.storage.deleteMetadata(LEGACY_CONFIG_KEY)
|
|
177
179
|
this.hasMigrated = true;
|
|
178
|
-
this.lastConfigVersion =
|
|
179
|
-
return
|
|
180
|
+
this.lastConfigVersion = config.version;
|
|
181
|
+
return config;
|
|
180
182
|
}
|
|
181
183
|
}
|
|
182
184
|
catch (error) {
|
|
@@ -217,6 +217,7 @@ export class ShardMigrationManager extends EventEmitter {
|
|
|
217
217
|
// Don't delete immediately in case of rollback
|
|
218
218
|
const cleanupKey = `cleanup:${shardId}:${Date.now()}`;
|
|
219
219
|
await this.storage.saveMetadata(cleanupKey, {
|
|
220
|
+
noun: 'Document',
|
|
220
221
|
shardId,
|
|
221
222
|
scheduledFor: Date.now() + 3600000 // Delete after 1 hour
|
|
222
223
|
});
|
|
@@ -236,6 +237,7 @@ export class ShardMigrationManager extends EventEmitter {
|
|
|
236
237
|
}
|
|
237
238
|
// Track progress
|
|
238
239
|
const progress = {
|
|
240
|
+
noun: 'Document',
|
|
239
241
|
migrationId: data.migrationId,
|
|
240
242
|
shardId: data.shardId,
|
|
241
243
|
received: data.offset + data.items.length,
|
|
@@ -163,7 +163,7 @@ export class StorageDiscovery extends EventEmitter {
|
|
|
163
163
|
// Remove ourselves from node registry
|
|
164
164
|
try {
|
|
165
165
|
// Mark as deleted rather than actually deleting
|
|
166
|
-
const deadNode = { ...this.nodeInfo, lastSeen: 0, status: 'inactive' };
|
|
166
|
+
const deadNode = { noun: 'Document', ...this.nodeInfo, lastSeen: 0, status: 'inactive' };
|
|
167
167
|
await this.storage.saveMetadata(`${this.CLUSTER_PATH}/nodes/${this.nodeId}.json`, deadNode);
|
|
168
168
|
}
|
|
169
169
|
catch (err) {
|
|
@@ -181,7 +181,7 @@ export class StorageDiscovery extends EventEmitter {
|
|
|
181
181
|
*/
|
|
182
182
|
async registerNode() {
|
|
183
183
|
const path = `${this.CLUSTER_PATH}/nodes/${this.nodeId}.json`;
|
|
184
|
-
await this.storage.saveMetadata(path, this.nodeInfo);
|
|
184
|
+
await this.storage.saveMetadata(path, { noun: 'Document', ...this.nodeInfo });
|
|
185
185
|
// Also update registry
|
|
186
186
|
await this.updateNodeRegistry(this.nodeId);
|
|
187
187
|
}
|
|
@@ -235,7 +235,8 @@ export class StorageDiscovery extends EventEmitter {
|
|
|
235
235
|
if (nodeId === this.nodeId)
|
|
236
236
|
continue;
|
|
237
237
|
try {
|
|
238
|
-
const
|
|
238
|
+
const nodeInfoData = await this.storage.getMetadata(`${this.CLUSTER_PATH}/nodes/${nodeId}.json`);
|
|
239
|
+
const nodeInfo = nodeInfoData;
|
|
239
240
|
// Check if node is alive
|
|
240
241
|
if (now - nodeInfo.lastSeen < this.NODE_TIMEOUT) {
|
|
241
242
|
if (!this.clusterConfig.nodes[nodeId]) {
|
|
@@ -291,6 +292,7 @@ export class StorageDiscovery extends EventEmitter {
|
|
|
291
292
|
registry = registry.filter(id => id !== remove);
|
|
292
293
|
}
|
|
293
294
|
await this.storage.saveMetadata(`${this.CLUSTER_PATH}/registry.json`, {
|
|
295
|
+
noun: 'Document',
|
|
294
296
|
nodes: registry,
|
|
295
297
|
updated: Date.now()
|
|
296
298
|
});
|
|
@@ -348,7 +350,7 @@ export class StorageDiscovery extends EventEmitter {
|
|
|
348
350
|
async saveClusterConfig() {
|
|
349
351
|
if (!this.clusterConfig)
|
|
350
352
|
return;
|
|
351
|
-
await this.storage.saveMetadata(`${this.CLUSTER_PATH}/config.json`, this.clusterConfig);
|
|
353
|
+
await this.storage.saveMetadata(`${this.CLUSTER_PATH}/config.json`, { noun: 'Document', ...this.clusterConfig });
|
|
352
354
|
}
|
|
353
355
|
/**
|
|
354
356
|
* Trigger leader election (simplified - not full Raft)
|
|
@@ -52,7 +52,7 @@ export declare class EmbeddingManager {
|
|
|
52
52
|
/**
|
|
53
53
|
* Generate embeddings
|
|
54
54
|
*/
|
|
55
|
-
embed(text: string | string[]): Promise<Vector>;
|
|
55
|
+
embed(text: string | string[] | Record<string, unknown>): Promise<Vector>;
|
|
56
56
|
/**
|
|
57
57
|
* Generate mock embeddings for unit tests
|
|
58
58
|
*/
|
|
@@ -94,7 +94,7 @@ export declare const embeddingManager: EmbeddingManager;
|
|
|
94
94
|
/**
|
|
95
95
|
* Direct embed function
|
|
96
96
|
*/
|
|
97
|
-
export declare function embed(text: string | string[]): Promise<Vector>;
|
|
97
|
+
export declare function embed(text: string | string[] | Record<string, unknown>): Promise<Vector>;
|
|
98
98
|
/**
|
|
99
99
|
* Get embedding function for compatibility
|
|
100
100
|
*/
|
|
@@ -174,9 +174,13 @@ export class EmbeddingManager {
|
|
|
174
174
|
else if (typeof text === 'string') {
|
|
175
175
|
input = text;
|
|
176
176
|
}
|
|
177
|
+
else if (typeof text === 'object') {
|
|
178
|
+
// Convert object to string representation
|
|
179
|
+
input = JSON.stringify(text);
|
|
180
|
+
}
|
|
177
181
|
else {
|
|
178
182
|
// This shouldn't happen but let's be defensive
|
|
179
|
-
console.warn('EmbeddingManager.embed received
|
|
183
|
+
console.warn('EmbeddingManager.embed received unexpected input type:', typeof text);
|
|
180
184
|
input = String(text);
|
|
181
185
|
}
|
|
182
186
|
// Generate embedding
|
|
@@ -218,8 +218,11 @@ export class LSMTree {
|
|
|
218
218
|
const data = sstable.serialize();
|
|
219
219
|
const storageKey = `${this.config.storagePrefix}-${sstable.metadata.id}`;
|
|
220
220
|
await this.storage.saveMetadata(storageKey, {
|
|
221
|
-
|
|
222
|
-
data:
|
|
221
|
+
noun: 'thing', // Required for NounMetadata
|
|
222
|
+
data: {
|
|
223
|
+
type: 'lsm-sstable',
|
|
224
|
+
data: Array.from(data) // Convert Uint8Array to number[] for JSON storage
|
|
225
|
+
}
|
|
223
226
|
});
|
|
224
227
|
// Add to L0 SSTables
|
|
225
228
|
if (!this.sstablesByLevel.has(0)) {
|
|
@@ -269,8 +272,11 @@ export class LSMTree {
|
|
|
269
272
|
const data = merged.serialize();
|
|
270
273
|
const storageKey = `${this.config.storagePrefix}-${merged.metadata.id}`;
|
|
271
274
|
await this.storage.saveMetadata(storageKey, {
|
|
272
|
-
|
|
273
|
-
data:
|
|
275
|
+
noun: 'thing', // Required for NounMetadata
|
|
276
|
+
data: {
|
|
277
|
+
type: 'lsm-sstable',
|
|
278
|
+
data: Array.from(data)
|
|
279
|
+
}
|
|
274
280
|
});
|
|
275
281
|
// Delete old SSTables from storage
|
|
276
282
|
for (const sstable of sstables) {
|
|
@@ -339,8 +345,9 @@ export class LSMTree {
|
|
|
339
345
|
*/
|
|
340
346
|
async loadManifest() {
|
|
341
347
|
try {
|
|
342
|
-
const
|
|
343
|
-
if (data) {
|
|
348
|
+
const metadata = await this.storage.getMetadata(`${this.config.storagePrefix}-manifest`);
|
|
349
|
+
if (metadata && metadata.data) {
|
|
350
|
+
const data = metadata.data;
|
|
344
351
|
this.manifest.sstables = new Map(Object.entries(data.sstables || {}));
|
|
345
352
|
this.manifest.lastCompaction = data.lastCompaction || Date.now();
|
|
346
353
|
this.manifest.totalRelationships = data.totalRelationships || 0;
|
|
@@ -361,15 +368,18 @@ export class LSMTree {
|
|
|
361
368
|
const loadPromise = (async () => {
|
|
362
369
|
try {
|
|
363
370
|
const storageKey = `${this.config.storagePrefix}-${sstableId}`;
|
|
364
|
-
const
|
|
365
|
-
if (
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
+
const metadata = await this.storage.getMetadata(storageKey);
|
|
372
|
+
if (metadata && metadata.data) {
|
|
373
|
+
const data = metadata.data;
|
|
374
|
+
if (data.type === 'lsm-sstable') {
|
|
375
|
+
// Convert number[] back to Uint8Array
|
|
376
|
+
const uint8Data = new Uint8Array(data.data);
|
|
377
|
+
const sstable = SSTable.deserialize(uint8Data);
|
|
378
|
+
if (!this.sstablesByLevel.has(level)) {
|
|
379
|
+
this.sstablesByLevel.set(level, []);
|
|
380
|
+
}
|
|
381
|
+
this.sstablesByLevel.get(level).push(sstable);
|
|
371
382
|
}
|
|
372
|
-
this.sstablesByLevel.get(level).push(sstable);
|
|
373
383
|
}
|
|
374
384
|
}
|
|
375
385
|
catch (error) {
|
|
@@ -386,12 +396,14 @@ export class LSMTree {
|
|
|
386
396
|
*/
|
|
387
397
|
async saveManifest() {
|
|
388
398
|
try {
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
399
|
+
await this.storage.saveMetadata(`${this.config.storagePrefix}-manifest`, {
|
|
400
|
+
noun: 'thing', // Required for NounMetadata
|
|
401
|
+
data: {
|
|
402
|
+
sstables: Object.fromEntries(this.manifest.sstables),
|
|
403
|
+
lastCompaction: this.manifest.lastCompaction,
|
|
404
|
+
totalRelationships: this.manifest.totalRelationships
|
|
405
|
+
}
|
|
406
|
+
});
|
|
395
407
|
}
|
|
396
408
|
catch (error) {
|
|
397
409
|
prodLog.error('LSMTree: Failed to save manifest', error);
|
|
@@ -332,8 +332,12 @@ export class TypeAwareHNSWIndex {
|
|
|
332
332
|
// Route each noun to its type index
|
|
333
333
|
for (const nounData of result.items) {
|
|
334
334
|
try {
|
|
335
|
-
//
|
|
336
|
-
|
|
335
|
+
// v4.0.0: Load metadata separately to get noun type
|
|
336
|
+
let nounType = nounData.nounType;
|
|
337
|
+
if (!nounType) {
|
|
338
|
+
const metadata = await this.storage.getNounMetadata(nounData.id);
|
|
339
|
+
nounType = (metadata?.noun || metadata?.type);
|
|
340
|
+
}
|
|
337
341
|
// Skip if type not in rebuild list
|
|
338
342
|
if (!nounType || !typesToRebuild.includes(nounType)) {
|
|
339
343
|
continue;
|