@soulcraft/brainy 3.1.1 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/augmentations/cacheAugmentation.d.ts +1 -0
- package/dist/augmentations/cacheAugmentation.js +5 -0
- package/dist/augmentations/display/types.d.ts +2 -0
- package/dist/augmentations/metricsAugmentation.d.ts +20 -4
- package/dist/augmentations/metricsAugmentation.js +19 -4
- package/dist/augmentations/universalDisplayAugmentation.js +4 -0
- package/dist/brainy.d.ts +77 -0
- package/dist/brainy.js +303 -26
- package/dist/graph/graphAdjacencyIndex.d.ts +24 -1
- package/dist/graph/graphAdjacencyIndex.js +54 -1
- package/dist/utils/logger.d.ts +1 -0
- package/dist/utils/logger.js +6 -1
- package/dist/utils/metadataIndex.d.ts +18 -1
- package/dist/utils/metadataIndex.js +39 -1
- package/package.json +1 -1
|
@@ -65,6 +65,11 @@ export class CacheAugmentation extends BaseAugmentation {
|
|
|
65
65
|
type: 'boolean',
|
|
66
66
|
default: true,
|
|
67
67
|
description: 'Automatically invalidate cache on data modifications'
|
|
68
|
+
},
|
|
69
|
+
silent: {
|
|
70
|
+
type: 'boolean',
|
|
71
|
+
default: false,
|
|
72
|
+
description: 'Suppress all console output'
|
|
68
73
|
}
|
|
69
74
|
},
|
|
70
75
|
additionalProperties: false
|
|
@@ -18,6 +18,8 @@ export interface DisplayConfig {
|
|
|
18
18
|
batchSize: number;
|
|
19
19
|
/** Minimum confidence threshold for AI type detection */
|
|
20
20
|
confidenceThreshold: number;
|
|
21
|
+
/** Silent mode - suppress all console output */
|
|
22
|
+
silent?: boolean;
|
|
21
23
|
/** Custom field mappings (userField -> displayField) */
|
|
22
24
|
customFieldMappings: Record<string, string>;
|
|
23
25
|
/** Type-specific priority fields for intelligent detection */
|
|
@@ -1,11 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Metrics Augmentation - Optional Performance & Usage Metrics
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* IMPORTANT: This is SEPARATE from core counting (brain.counts.*) which is always enabled.
|
|
5
|
+
*
|
|
6
|
+
* Core counting provides O(1) entity/relationship counts for scalability.
|
|
7
|
+
* This augmentation provides performance analytics and observability.
|
|
8
|
+
*
|
|
9
|
+
* Features:
|
|
10
|
+
* - Search performance tracking (latency, throughput)
|
|
11
|
+
* - Cache hit/miss rates
|
|
12
|
+
* - Operation timing analysis
|
|
13
|
+
* - Usage pattern insights
|
|
6
14
|
*
|
|
7
15
|
* Zero-config: Automatically enabled for observability
|
|
8
|
-
* Can be disabled
|
|
16
|
+
* Can be disabled via augmentation registry without affecting core performance
|
|
9
17
|
*/
|
|
10
18
|
import { BaseAugmentation } from './brainyAugmentation.js';
|
|
11
19
|
export interface MetricsConfig {
|
|
@@ -16,6 +24,7 @@ export interface MetricsConfig {
|
|
|
16
24
|
trackStorageSizes?: boolean;
|
|
17
25
|
persistMetrics?: boolean;
|
|
18
26
|
metricsInterval?: number;
|
|
27
|
+
silent?: boolean;
|
|
19
28
|
}
|
|
20
29
|
/**
|
|
21
30
|
* MetricsAugmentation - Makes metrics collection optional and pluggable
|
|
@@ -71,10 +80,16 @@ export declare class MetricsAugmentation extends BaseAugmentation {
|
|
|
71
80
|
*/
|
|
72
81
|
private persistMetrics;
|
|
73
82
|
/**
|
|
74
|
-
* Get current metrics
|
|
83
|
+
* Get current metrics (performance analytics only)
|
|
84
|
+
*
|
|
85
|
+
* NOTE: For production counting (entities, relationships), use:
|
|
86
|
+
* - brain.counts.entities() - O(1) total count
|
|
87
|
+
* - brain.counts.byType() - O(1) type-specific counts
|
|
88
|
+
* - brain.counts.relationships() - O(1) relationship counts
|
|
75
89
|
*/
|
|
76
90
|
getStatistics(): {
|
|
77
91
|
enabled: boolean;
|
|
92
|
+
note: string;
|
|
78
93
|
totalSearches: number;
|
|
79
94
|
totalUpdates: number;
|
|
80
95
|
contentTypes: {};
|
|
@@ -166,6 +181,7 @@ export declare class MetricsAugmentation extends BaseAugmentation {
|
|
|
166
181
|
lastUpdated?: string | undefined;
|
|
167
182
|
distributedConfig?: import("../types/distributedTypes.js").SharedConfig | undefined;
|
|
168
183
|
enabled: boolean;
|
|
184
|
+
note: string;
|
|
169
185
|
totalSearches?: undefined;
|
|
170
186
|
totalUpdates?: undefined;
|
|
171
187
|
verbTypes?: undefined;
|
|
@@ -1,11 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Metrics Augmentation - Optional Performance & Usage Metrics
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* IMPORTANT: This is SEPARATE from core counting (brain.counts.*) which is always enabled.
|
|
5
|
+
*
|
|
6
|
+
* Core counting provides O(1) entity/relationship counts for scalability.
|
|
7
|
+
* This augmentation provides performance analytics and observability.
|
|
8
|
+
*
|
|
9
|
+
* Features:
|
|
10
|
+
* - Search performance tracking (latency, throughput)
|
|
11
|
+
* - Cache hit/miss rates
|
|
12
|
+
* - Operation timing analysis
|
|
13
|
+
* - Usage pattern insights
|
|
6
14
|
*
|
|
7
15
|
* Zero-config: Automatically enabled for observability
|
|
8
|
-
* Can be disabled
|
|
16
|
+
* Can be disabled via augmentation registry without affecting core performance
|
|
9
17
|
*/
|
|
10
18
|
import { BaseAugmentation } from './brainyAugmentation.js';
|
|
11
19
|
import { StatisticsCollector } from '../utils/statisticsCollector.js';
|
|
@@ -224,12 +232,18 @@ export class MetricsAugmentation extends BaseAugmentation {
|
|
|
224
232
|
}
|
|
225
233
|
}
|
|
226
234
|
/**
|
|
227
|
-
* Get current metrics
|
|
235
|
+
* Get current metrics (performance analytics only)
|
|
236
|
+
*
|
|
237
|
+
* NOTE: For production counting (entities, relationships), use:
|
|
238
|
+
* - brain.counts.entities() - O(1) total count
|
|
239
|
+
* - brain.counts.byType() - O(1) type-specific counts
|
|
240
|
+
* - brain.counts.relationships() - O(1) relationship counts
|
|
228
241
|
*/
|
|
229
242
|
getStatistics() {
|
|
230
243
|
if (!this.statisticsCollector) {
|
|
231
244
|
return {
|
|
232
245
|
enabled: false,
|
|
246
|
+
note: 'For core counting, use brain.counts.* APIs which are always available',
|
|
233
247
|
totalSearches: 0,
|
|
234
248
|
totalUpdates: 0,
|
|
235
249
|
contentTypes: {},
|
|
@@ -243,6 +257,7 @@ export class MetricsAugmentation extends BaseAugmentation {
|
|
|
243
257
|
}
|
|
244
258
|
return {
|
|
245
259
|
enabled: true,
|
|
260
|
+
note: 'Performance analytics only. Core counting available via brain.counts.*',
|
|
246
261
|
...this.statisticsCollector.getStatistics()
|
|
247
262
|
};
|
|
248
263
|
}
|
|
@@ -234,6 +234,10 @@ export class UniversalDisplayAugmentation extends BaseAugmentation {
|
|
|
234
234
|
*/
|
|
235
235
|
createExploreMethod(entity) {
|
|
236
236
|
return async () => {
|
|
237
|
+
// Respect silent mode
|
|
238
|
+
if (this.config.silent) {
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
237
241
|
console.log(`\n📋 Entity Exploration: ${entity.id || 'unknown'}`);
|
|
238
242
|
console.log('━'.repeat(50));
|
|
239
243
|
// Show user data
|
package/dist/brainy.d.ts
CHANGED
|
@@ -22,6 +22,7 @@ export declare class Brainy<T = any> {
|
|
|
22
22
|
private distance;
|
|
23
23
|
private augmentationRegistry;
|
|
24
24
|
private config;
|
|
25
|
+
private originalConsole?;
|
|
25
26
|
private _neural?;
|
|
26
27
|
private _nlp?;
|
|
27
28
|
private _tripleIntelligence?;
|
|
@@ -195,6 +196,82 @@ export declare class Brainy<T = any> {
|
|
|
195
196
|
services: string[];
|
|
196
197
|
density: number;
|
|
197
198
|
}>;
|
|
199
|
+
/**
|
|
200
|
+
* Efficient Pagination API - Production-scale pagination using index-first approach
|
|
201
|
+
* Automatically optimizes based on query type and applies pagination at the index level
|
|
202
|
+
*/
|
|
203
|
+
get pagination(): {
|
|
204
|
+
find: (params: FindParams<T> & {
|
|
205
|
+
page?: number;
|
|
206
|
+
pageSize?: number;
|
|
207
|
+
}) => Promise<Result<T>[]>;
|
|
208
|
+
count: (params: Omit<FindParams<T>, "limit" | "offset">) => Promise<number>;
|
|
209
|
+
meta: (params: FindParams<T> & {
|
|
210
|
+
page?: number;
|
|
211
|
+
pageSize?: number;
|
|
212
|
+
}) => Promise<{
|
|
213
|
+
page: number;
|
|
214
|
+
pageSize: number;
|
|
215
|
+
totalCount: number;
|
|
216
|
+
totalPages: number;
|
|
217
|
+
hasNext: boolean;
|
|
218
|
+
hasPrev: boolean;
|
|
219
|
+
}>;
|
|
220
|
+
};
|
|
221
|
+
/**
|
|
222
|
+
* Streaming API - Process millions of entities with constant memory using existing Pipeline
|
|
223
|
+
* Integrates with index-based optimizations for maximum efficiency
|
|
224
|
+
*/
|
|
225
|
+
get streaming(): {
|
|
226
|
+
entities: (filter?: Partial<FindParams<T>>) => AsyncGenerator<Entity<T>>;
|
|
227
|
+
search: (params: FindParams<T>, batchSize?: number) => AsyncGenerator<{
|
|
228
|
+
id: string;
|
|
229
|
+
score: number;
|
|
230
|
+
entity: Entity<T>;
|
|
231
|
+
}>;
|
|
232
|
+
relationships: (filter?: {
|
|
233
|
+
type?: string;
|
|
234
|
+
sourceId?: string;
|
|
235
|
+
targetId?: string;
|
|
236
|
+
}) => AsyncGenerator<any>;
|
|
237
|
+
pipeline: (source: AsyncIterable<any>) => any;
|
|
238
|
+
process: (processor: (entity: Entity<T>) => Promise<Entity<T>>, filter?: Partial<FindParams<T>>, options?: {
|
|
239
|
+
batchSize: number;
|
|
240
|
+
parallel: number;
|
|
241
|
+
}) => Promise<void>;
|
|
242
|
+
};
|
|
243
|
+
/**
|
|
244
|
+
* O(1) Count API - Production-scale counting using existing indexes
|
|
245
|
+
* Works across all storage adapters (FileSystem, OPFS, S3, Memory)
|
|
246
|
+
*/
|
|
247
|
+
get counts(): {
|
|
248
|
+
entities: () => number;
|
|
249
|
+
relationships: () => number;
|
|
250
|
+
byType: (type?: string) => number | {
|
|
251
|
+
[k: string]: number;
|
|
252
|
+
};
|
|
253
|
+
byRelationshipType: (type?: string) => number | {
|
|
254
|
+
[k: string]: number;
|
|
255
|
+
};
|
|
256
|
+
byCriteria: (field: string, value: any) => Promise<number>;
|
|
257
|
+
getAllTypeCounts: () => Map<string, number>;
|
|
258
|
+
getStats: () => {
|
|
259
|
+
entities: {
|
|
260
|
+
total: number;
|
|
261
|
+
byType: {
|
|
262
|
+
[k: string]: number;
|
|
263
|
+
};
|
|
264
|
+
};
|
|
265
|
+
relationships: {
|
|
266
|
+
totalRelationships: number;
|
|
267
|
+
relationshipsByType: Record<string, number>;
|
|
268
|
+
uniqueSourceNodes: number;
|
|
269
|
+
uniqueTargetNodes: number;
|
|
270
|
+
totalNodes: number;
|
|
271
|
+
};
|
|
272
|
+
density: number;
|
|
273
|
+
};
|
|
274
|
+
};
|
|
198
275
|
/**
|
|
199
276
|
* Augmentations API - Clean and simple
|
|
200
277
|
*/
|
package/dist/brainy.js
CHANGED
|
@@ -16,6 +16,7 @@ import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js';
|
|
|
16
16
|
import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js';
|
|
17
17
|
import { MetadataIndexManager } from './utils/metadataIndex.js';
|
|
18
18
|
import { GraphAdjacencyIndex } from './graph/graphAdjacencyIndex.js';
|
|
19
|
+
import { createPipeline } from './streaming/pipeline.js';
|
|
19
20
|
import { configureLogger, LogLevel } from './utils/logger.js';
|
|
20
21
|
import { NounType } from './types/graphTypes.js';
|
|
21
22
|
/**
|
|
@@ -62,7 +63,20 @@ export class Brainy {
|
|
|
62
63
|
}
|
|
63
64
|
// Configure logging based on config options
|
|
64
65
|
if (this.config.silent) {
|
|
65
|
-
|
|
66
|
+
// Store original console methods for restoration
|
|
67
|
+
this.originalConsole = {
|
|
68
|
+
log: console.log,
|
|
69
|
+
info: console.info,
|
|
70
|
+
warn: console.warn,
|
|
71
|
+
error: console.error
|
|
72
|
+
};
|
|
73
|
+
// Override all console methods to completely silence output
|
|
74
|
+
console.log = () => { };
|
|
75
|
+
console.info = () => { };
|
|
76
|
+
console.warn = () => { };
|
|
77
|
+
console.error = () => { };
|
|
78
|
+
// Also configure logger for silent mode
|
|
79
|
+
configureLogger({ level: LogLevel.SILENT }); // Suppress all logs
|
|
66
80
|
}
|
|
67
81
|
else if (this.config.verbose) {
|
|
68
82
|
configureLogger({ level: LogLevel.DEBUG }); // Enable verbose logging
|
|
@@ -468,22 +482,51 @@ export class Brainy {
|
|
|
468
482
|
const filteredIds = await this.metadataIndex.getIdsForFilter(filter);
|
|
469
483
|
// CRITICAL FIX: Handle both cases properly
|
|
470
484
|
if (results.length > 0) {
|
|
471
|
-
// Filter existing results (from vector search)
|
|
485
|
+
// OPTIMIZED: Filter existing results (from vector search) efficiently
|
|
472
486
|
const filteredIdSet = new Set(filteredIds);
|
|
473
487
|
results = results.filter((r) => filteredIdSet.has(r.id));
|
|
488
|
+
// Apply early pagination for vector + metadata queries
|
|
489
|
+
const limit = params.limit || 10;
|
|
490
|
+
const offset = params.offset || 0;
|
|
491
|
+
// If we have enough filtered results, sort and paginate early
|
|
492
|
+
if (results.length >= offset + limit) {
|
|
493
|
+
results.sort((a, b) => b.score - a.score);
|
|
494
|
+
results = results.slice(offset, offset + limit);
|
|
495
|
+
// Load entities only for the paginated results
|
|
496
|
+
for (const result of results) {
|
|
497
|
+
if (!result.entity) {
|
|
498
|
+
const entity = await this.get(result.id);
|
|
499
|
+
if (entity) {
|
|
500
|
+
result.entity = entity;
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
// Early return if no other processing needed
|
|
505
|
+
if (!params.connected && !params.fusion) {
|
|
506
|
+
return results;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
474
509
|
}
|
|
475
510
|
else {
|
|
476
|
-
//
|
|
477
|
-
|
|
511
|
+
// OPTIMIZED: Apply pagination to filtered IDs BEFORE loading entities
|
|
512
|
+
const limit = params.limit || 10;
|
|
513
|
+
const offset = params.offset || 0;
|
|
514
|
+
const pageIds = filteredIds.slice(offset, offset + limit);
|
|
515
|
+
// Load only entities for current page - O(page_size) instead of O(total_results)
|
|
516
|
+
for (const id of pageIds) {
|
|
478
517
|
const entity = await this.get(id);
|
|
479
518
|
if (entity) {
|
|
480
519
|
results.push({
|
|
481
520
|
id,
|
|
482
521
|
score: 1.0, // All metadata matches are equally relevant
|
|
483
|
-
entity
|
|
522
|
+
entity: entity
|
|
484
523
|
});
|
|
485
524
|
}
|
|
486
525
|
}
|
|
526
|
+
// Early return for metadata-only queries with pagination applied
|
|
527
|
+
if (!params.query && !params.connected) {
|
|
528
|
+
return results;
|
|
529
|
+
}
|
|
487
530
|
}
|
|
488
531
|
}
|
|
489
532
|
// Graph search component with O(1) traversal
|
|
@@ -494,10 +537,11 @@ export class Brainy {
|
|
|
494
537
|
if (params.fusion && results.length > 0) {
|
|
495
538
|
results = this.applyFusionScoring(results, params.fusion);
|
|
496
539
|
}
|
|
497
|
-
// Sort
|
|
540
|
+
// OPTIMIZED: Sort first, then apply efficient pagination
|
|
498
541
|
results.sort((a, b) => b.score - a.score);
|
|
499
542
|
const limit = params.limit || 10;
|
|
500
543
|
const offset = params.offset || 0;
|
|
544
|
+
// Efficient pagination - only slice what we need
|
|
501
545
|
return results.slice(offset, offset + limit);
|
|
502
546
|
});
|
|
503
547
|
// Record performance for auto-tuning
|
|
@@ -863,24 +907,16 @@ export class Brainy {
|
|
|
863
907
|
*/
|
|
864
908
|
async insights() {
|
|
865
909
|
await this.ensureInitialized();
|
|
866
|
-
//
|
|
867
|
-
const
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
const
|
|
871
|
-
//
|
|
872
|
-
const
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
const
|
|
876
|
-
// Count by type
|
|
877
|
-
const types = {};
|
|
878
|
-
for (const entity of entitiesResult.items) {
|
|
879
|
-
const type = entity.metadata?.noun || 'unknown';
|
|
880
|
-
types[type] = (types[type] || 0) + 1;
|
|
881
|
-
}
|
|
882
|
-
// Get unique services
|
|
883
|
-
const services = [...new Set(entitiesResult.items.map((e) => e.metadata?.service).filter(Boolean))];
|
|
910
|
+
// O(1) entity counting using existing MetadataIndexManager
|
|
911
|
+
const entities = this.metadataIndex.getTotalEntityCount();
|
|
912
|
+
// O(1) count by type using existing index tracking
|
|
913
|
+
const typeCountsMap = this.metadataIndex.getAllEntityCounts();
|
|
914
|
+
const types = Object.fromEntries(typeCountsMap);
|
|
915
|
+
// O(1) relationships count using GraphAdjacencyIndex
|
|
916
|
+
const relationships = this.graphIndex.getTotalRelationshipCount();
|
|
917
|
+
// Get unique services - O(log n) using index
|
|
918
|
+
const serviceValues = await this.metadataIndex.getFilterValues('service');
|
|
919
|
+
const services = serviceValues.filter(Boolean);
|
|
884
920
|
// Calculate density (relationships per entity)
|
|
885
921
|
const density = entities > 0 ? relationships / entities : 0;
|
|
886
922
|
return {
|
|
@@ -891,6 +927,229 @@ export class Brainy {
|
|
|
891
927
|
density
|
|
892
928
|
};
|
|
893
929
|
}
|
|
930
|
+
/**
|
|
931
|
+
* Efficient Pagination API - Production-scale pagination using index-first approach
|
|
932
|
+
* Automatically optimizes based on query type and applies pagination at the index level
|
|
933
|
+
*/
|
|
934
|
+
get pagination() {
|
|
935
|
+
return {
|
|
936
|
+
// Get paginated results with automatic optimization
|
|
937
|
+
find: async (params) => {
|
|
938
|
+
const page = params.page || 1;
|
|
939
|
+
const pageSize = params.pageSize || 10;
|
|
940
|
+
const offset = (page - 1) * pageSize;
|
|
941
|
+
return this.find({
|
|
942
|
+
...params,
|
|
943
|
+
limit: pageSize,
|
|
944
|
+
offset
|
|
945
|
+
});
|
|
946
|
+
},
|
|
947
|
+
// Get total count for pagination UI (O(1) when possible)
|
|
948
|
+
count: async (params) => {
|
|
949
|
+
// For simple type queries, use O(1) index counting
|
|
950
|
+
if (params.type && !params.query && !params.where && !params.connected) {
|
|
951
|
+
const types = Array.isArray(params.type) ? params.type : [params.type];
|
|
952
|
+
return types.reduce((sum, type) => sum + this.metadataIndex.getEntityCountByType(type), 0);
|
|
953
|
+
}
|
|
954
|
+
// For complex queries, use metadata index for efficient counting
|
|
955
|
+
if (params.where || params.service) {
|
|
956
|
+
let filter = {};
|
|
957
|
+
if (params.where)
|
|
958
|
+
Object.assign(filter, params.where);
|
|
959
|
+
if (params.service)
|
|
960
|
+
filter.service = params.service;
|
|
961
|
+
if (params.type) {
|
|
962
|
+
const types = Array.isArray(params.type) ? params.type : [params.type];
|
|
963
|
+
if (types.length === 1) {
|
|
964
|
+
filter.noun = types[0];
|
|
965
|
+
}
|
|
966
|
+
else {
|
|
967
|
+
const baseFilter = { ...filter };
|
|
968
|
+
filter = {
|
|
969
|
+
anyOf: types.map(type => ({ noun: type, ...baseFilter }))
|
|
970
|
+
};
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
const filteredIds = await this.metadataIndex.getIdsForFilter(filter);
|
|
974
|
+
return filteredIds.length;
|
|
975
|
+
}
|
|
976
|
+
// Fallback: total entity count
|
|
977
|
+
return this.metadataIndex.getTotalEntityCount();
|
|
978
|
+
},
|
|
979
|
+
// Get pagination metadata
|
|
980
|
+
meta: async (params) => {
|
|
981
|
+
const page = params.page || 1;
|
|
982
|
+
const pageSize = params.pageSize || 10;
|
|
983
|
+
const totalCount = await this.pagination.count(params);
|
|
984
|
+
const totalPages = Math.ceil(totalCount / pageSize);
|
|
985
|
+
return {
|
|
986
|
+
page,
|
|
987
|
+
pageSize,
|
|
988
|
+
totalCount,
|
|
989
|
+
totalPages,
|
|
990
|
+
hasNext: page < totalPages,
|
|
991
|
+
hasPrev: page > 1
|
|
992
|
+
};
|
|
993
|
+
}
|
|
994
|
+
};
|
|
995
|
+
}
|
|
996
|
+
/**
|
|
997
|
+
* Streaming API - Process millions of entities with constant memory using existing Pipeline
|
|
998
|
+
* Integrates with index-based optimizations for maximum efficiency
|
|
999
|
+
*/
|
|
1000
|
+
get streaming() {
|
|
1001
|
+
return {
|
|
1002
|
+
// Stream all entities with optional filtering
|
|
1003
|
+
entities: async function* (filter) {
|
|
1004
|
+
if (filter?.type || filter?.where || filter?.service) {
|
|
1005
|
+
// Use MetadataIndexManager for efficient filtered streaming
|
|
1006
|
+
let filterObj = {};
|
|
1007
|
+
if (filter.where)
|
|
1008
|
+
Object.assign(filterObj, filter.where);
|
|
1009
|
+
if (filter.service)
|
|
1010
|
+
filterObj.service = filter.service;
|
|
1011
|
+
if (filter.type) {
|
|
1012
|
+
const types = Array.isArray(filter.type) ? filter.type : [filter.type];
|
|
1013
|
+
if (types.length === 1) {
|
|
1014
|
+
filterObj.noun = types[0];
|
|
1015
|
+
}
|
|
1016
|
+
else {
|
|
1017
|
+
const baseFilterObj = { ...filterObj };
|
|
1018
|
+
filterObj = {
|
|
1019
|
+
anyOf: types.map(type => ({ noun: type, ...baseFilterObj }))
|
|
1020
|
+
};
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
const filteredIds = await this.metadataIndex.getIdsForFilter(filterObj);
|
|
1024
|
+
// Stream filtered entities in batches for memory efficiency
|
|
1025
|
+
const batchSize = 100;
|
|
1026
|
+
for (let i = 0; i < filteredIds.length; i += batchSize) {
|
|
1027
|
+
const batchIds = filteredIds.slice(i, i + batchSize);
|
|
1028
|
+
for (const id of batchIds) {
|
|
1029
|
+
const entity = await this.get(id);
|
|
1030
|
+
if (entity)
|
|
1031
|
+
yield entity;
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
else {
|
|
1036
|
+
// Stream all entities using storage adapter pagination
|
|
1037
|
+
let offset = 0;
|
|
1038
|
+
const batchSize = 100;
|
|
1039
|
+
let hasMore = true;
|
|
1040
|
+
while (hasMore) {
|
|
1041
|
+
const result = await this.storage.getNouns({
|
|
1042
|
+
pagination: { offset, limit: batchSize }
|
|
1043
|
+
});
|
|
1044
|
+
for (const noun of result.items) {
|
|
1045
|
+
// Convert HNSWNoun to Entity<T>
|
|
1046
|
+
yield noun;
|
|
1047
|
+
}
|
|
1048
|
+
hasMore = result.hasMore;
|
|
1049
|
+
offset += batchSize;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
}.bind(this),
|
|
1053
|
+
// Stream search results efficiently
|
|
1054
|
+
search: async function* (params, batchSize = 50) {
|
|
1055
|
+
const originalLimit = params.limit;
|
|
1056
|
+
let offset = 0;
|
|
1057
|
+
let hasMore = true;
|
|
1058
|
+
while (hasMore) {
|
|
1059
|
+
const batchResults = await this.find({
|
|
1060
|
+
...params,
|
|
1061
|
+
limit: batchSize,
|
|
1062
|
+
offset
|
|
1063
|
+
});
|
|
1064
|
+
for (const result of batchResults) {
|
|
1065
|
+
yield result;
|
|
1066
|
+
}
|
|
1067
|
+
hasMore = batchResults.length === batchSize;
|
|
1068
|
+
offset += batchSize;
|
|
1069
|
+
// Respect original limit if specified
|
|
1070
|
+
if (originalLimit && offset >= originalLimit) {
|
|
1071
|
+
break;
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
}.bind(this),
|
|
1075
|
+
// Stream relationships efficiently
|
|
1076
|
+
relationships: async function* (filter) {
|
|
1077
|
+
let offset = 0;
|
|
1078
|
+
const batchSize = 100;
|
|
1079
|
+
let hasMore = true;
|
|
1080
|
+
while (hasMore) {
|
|
1081
|
+
const result = await this.storage.getVerbs({
|
|
1082
|
+
pagination: { offset, limit: batchSize },
|
|
1083
|
+
filter
|
|
1084
|
+
});
|
|
1085
|
+
for (const verb of result.items) {
|
|
1086
|
+
yield verb;
|
|
1087
|
+
}
|
|
1088
|
+
hasMore = result.hasMore;
|
|
1089
|
+
offset += batchSize;
|
|
1090
|
+
}
|
|
1091
|
+
}.bind(this),
|
|
1092
|
+
// Create processing pipeline from stream
|
|
1093
|
+
pipeline: (source) => {
|
|
1094
|
+
return createPipeline(this).source(source);
|
|
1095
|
+
},
|
|
1096
|
+
// Batch process entities with Pipeline system
|
|
1097
|
+
process: async function (processor, filter, options = { batchSize: 50, parallel: 4 }) {
|
|
1098
|
+
return createPipeline(this)
|
|
1099
|
+
.source(this.streaming.entities(filter))
|
|
1100
|
+
.batch(options.batchSize)
|
|
1101
|
+
.parallelSink(async (batch) => {
|
|
1102
|
+
await Promise.all(batch.map(processor));
|
|
1103
|
+
}, options.parallel)
|
|
1104
|
+
.run();
|
|
1105
|
+
}.bind(this)
|
|
1106
|
+
};
|
|
1107
|
+
}
|
|
1108
|
+
/**
|
|
1109
|
+
* O(1) Count API - Production-scale counting using existing indexes
|
|
1110
|
+
* Works across all storage adapters (FileSystem, OPFS, S3, Memory)
|
|
1111
|
+
*/
|
|
1112
|
+
get counts() {
|
|
1113
|
+
return {
|
|
1114
|
+
// O(1) total entity count
|
|
1115
|
+
entities: () => this.metadataIndex.getTotalEntityCount(),
|
|
1116
|
+
// O(1) total relationship count
|
|
1117
|
+
relationships: () => this.graphIndex.getTotalRelationshipCount(),
|
|
1118
|
+
// O(1) count by type
|
|
1119
|
+
byType: (type) => {
|
|
1120
|
+
if (type) {
|
|
1121
|
+
return this.metadataIndex.getEntityCountByType(type);
|
|
1122
|
+
}
|
|
1123
|
+
return Object.fromEntries(this.metadataIndex.getAllEntityCounts());
|
|
1124
|
+
},
|
|
1125
|
+
// O(1) count by relationship type
|
|
1126
|
+
byRelationshipType: (type) => {
|
|
1127
|
+
if (type) {
|
|
1128
|
+
return this.graphIndex.getRelationshipCountByType(type);
|
|
1129
|
+
}
|
|
1130
|
+
return Object.fromEntries(this.graphIndex.getAllRelationshipCounts());
|
|
1131
|
+
},
|
|
1132
|
+
// O(1) count by field-value criteria
|
|
1133
|
+
byCriteria: async (field, value) => {
|
|
1134
|
+
return this.metadataIndex.getCountForCriteria(field, value);
|
|
1135
|
+
},
|
|
1136
|
+
// Get all type counts as Map for performance-critical operations
|
|
1137
|
+
getAllTypeCounts: () => this.metadataIndex.getAllEntityCounts(),
|
|
1138
|
+
// Get complete statistics
|
|
1139
|
+
getStats: () => {
|
|
1140
|
+
const entityStats = {
|
|
1141
|
+
total: this.metadataIndex.getTotalEntityCount(),
|
|
1142
|
+
byType: Object.fromEntries(this.metadataIndex.getAllEntityCounts())
|
|
1143
|
+
};
|
|
1144
|
+
const relationshipStats = this.graphIndex.getRelationshipStats();
|
|
1145
|
+
return {
|
|
1146
|
+
entities: entityStats,
|
|
1147
|
+
relationships: relationshipStats,
|
|
1148
|
+
density: entityStats.total > 0 ? relationshipStats.totalRelationships / entityStats.total : 0
|
|
1149
|
+
};
|
|
1150
|
+
}
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
894
1153
|
/**
|
|
895
1154
|
* Augmentations API - Clean and simple
|
|
896
1155
|
*/
|
|
@@ -1162,8 +1421,18 @@ export class Brainy {
|
|
|
1162
1421
|
*/
|
|
1163
1422
|
setupAugmentations() {
|
|
1164
1423
|
const registry = new AugmentationRegistry();
|
|
1165
|
-
// Register default augmentations
|
|
1166
|
-
const
|
|
1424
|
+
// Register default augmentations with silent mode support
|
|
1425
|
+
const augmentationConfig = {
|
|
1426
|
+
...this.config.augmentations,
|
|
1427
|
+
// Pass silent mode to all augmentations
|
|
1428
|
+
...(this.config.silent && {
|
|
1429
|
+
cache: this.config.augmentations?.cache !== false ? { ...this.config.augmentations?.cache, silent: true } : false,
|
|
1430
|
+
metrics: this.config.augmentations?.metrics !== false ? { ...this.config.augmentations?.metrics, silent: true } : false,
|
|
1431
|
+
display: this.config.augmentations?.display !== false ? { ...this.config.augmentations?.display, silent: true } : false,
|
|
1432
|
+
monitoring: this.config.augmentations?.monitoring !== false ? { ...this.config.augmentations?.monitoring, silent: true } : false
|
|
1433
|
+
})
|
|
1434
|
+
};
|
|
1435
|
+
const defaults = createDefaultAugmentations(augmentationConfig);
|
|
1167
1436
|
for (const aug of defaults) {
|
|
1168
1437
|
registry.register(aug);
|
|
1169
1438
|
}
|
|
@@ -1242,6 +1511,14 @@ export class Brainy {
|
|
|
1242
1511
|
await aug.shutdown();
|
|
1243
1512
|
}
|
|
1244
1513
|
}
|
|
1514
|
+
// Restore console methods if silent mode was enabled
|
|
1515
|
+
if (this.config.silent && this.originalConsole) {
|
|
1516
|
+
console.log = this.originalConsole.log;
|
|
1517
|
+
console.info = this.originalConsole.info;
|
|
1518
|
+
console.warn = this.originalConsole.warn;
|
|
1519
|
+
console.error = this.originalConsole.error;
|
|
1520
|
+
this.originalConsole = undefined;
|
|
1521
|
+
}
|
|
1245
1522
|
// Storage doesn't have close in current interface
|
|
1246
1523
|
// We'll just mark as not initialized
|
|
1247
1524
|
this.initialized = false;
|
|
@@ -42,6 +42,7 @@ export declare class GraphAdjacencyIndex {
|
|
|
42
42
|
private flushTimer?;
|
|
43
43
|
private rebuildStartTime;
|
|
44
44
|
private totalRelationshipsIndexed;
|
|
45
|
+
private relationshipCountsByType;
|
|
45
46
|
constructor(storage: StorageAdapter, config?: GraphIndexConfig);
|
|
46
47
|
/**
|
|
47
48
|
* Core API - O(1) neighbor lookup
|
|
@@ -49,9 +50,31 @@ export declare class GraphAdjacencyIndex {
|
|
|
49
50
|
*/
|
|
50
51
|
getNeighbors(id: string, direction?: 'in' | 'out' | 'both'): Promise<string[]>;
|
|
51
52
|
/**
|
|
52
|
-
* Get relationship count
|
|
53
|
+
* Get total relationship count - O(1) operation
|
|
53
54
|
*/
|
|
54
55
|
size(): number;
|
|
56
|
+
/**
|
|
57
|
+
* Get relationship count by type - O(1) operation using existing tracking
|
|
58
|
+
*/
|
|
59
|
+
getRelationshipCountByType(type: string): number;
|
|
60
|
+
/**
|
|
61
|
+
* Get total relationship count - O(1) operation
|
|
62
|
+
*/
|
|
63
|
+
getTotalRelationshipCount(): number;
|
|
64
|
+
/**
|
|
65
|
+
* Get all relationship types and their counts - O(1) operation
|
|
66
|
+
*/
|
|
67
|
+
getAllRelationshipCounts(): Map<string, number>;
|
|
68
|
+
/**
|
|
69
|
+
* Get relationship statistics with enhanced counting information
|
|
70
|
+
*/
|
|
71
|
+
getRelationshipStats(): {
|
|
72
|
+
totalRelationships: number;
|
|
73
|
+
relationshipsByType: Record<string, number>;
|
|
74
|
+
uniqueSourceNodes: number;
|
|
75
|
+
uniqueTargetNodes: number;
|
|
76
|
+
totalNodes: number;
|
|
77
|
+
};
|
|
55
78
|
/**
|
|
56
79
|
* Add relationship to index - O(1) amortized
|
|
57
80
|
*/
|
|
@@ -28,6 +28,8 @@ export class GraphAdjacencyIndex {
|
|
|
28
28
|
this.isRebuilding = false;
|
|
29
29
|
this.rebuildStartTime = 0;
|
|
30
30
|
this.totalRelationshipsIndexed = 0;
|
|
31
|
+
// Production-scale relationship counting by type
|
|
32
|
+
this.relationshipCountsByType = new Map();
|
|
31
33
|
this.storage = storage;
|
|
32
34
|
this.config = {
|
|
33
35
|
maxIndexSize: config.maxIndexSize ?? 100000,
|
|
@@ -70,11 +72,50 @@ export class GraphAdjacencyIndex {
|
|
|
70
72
|
return result;
|
|
71
73
|
}
|
|
72
74
|
/**
|
|
73
|
-
* Get relationship count
|
|
75
|
+
* Get total relationship count - O(1) operation
|
|
74
76
|
*/
|
|
75
77
|
size() {
|
|
76
78
|
return this.verbIndex.size;
|
|
77
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* Get relationship count by type - O(1) operation using existing tracking
|
|
82
|
+
*/
|
|
83
|
+
getRelationshipCountByType(type) {
|
|
84
|
+
return this.relationshipCountsByType.get(type) || 0;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Get total relationship count - O(1) operation
|
|
88
|
+
*/
|
|
89
|
+
getTotalRelationshipCount() {
|
|
90
|
+
return this.verbIndex.size;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Get all relationship types and their counts - O(1) operation
|
|
94
|
+
*/
|
|
95
|
+
getAllRelationshipCounts() {
|
|
96
|
+
return new Map(this.relationshipCountsByType);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Get relationship statistics with enhanced counting information
|
|
100
|
+
*/
|
|
101
|
+
getRelationshipStats() {
|
|
102
|
+
const totalRelationships = this.verbIndex.size;
|
|
103
|
+
const relationshipsByType = Object.fromEntries(this.relationshipCountsByType);
|
|
104
|
+
const uniqueSourceNodes = this.sourceIndex.size;
|
|
105
|
+
const uniqueTargetNodes = this.targetIndex.size;
|
|
106
|
+
// Calculate total unique nodes (source ∪ target)
|
|
107
|
+
const allNodes = new Set();
|
|
108
|
+
this.sourceIndex.keys().forEach(id => allNodes.add(id));
|
|
109
|
+
this.targetIndex.keys().forEach(id => allNodes.add(id));
|
|
110
|
+
const totalNodes = allNodes.size;
|
|
111
|
+
return {
|
|
112
|
+
totalRelationships,
|
|
113
|
+
relationshipsByType,
|
|
114
|
+
uniqueSourceNodes,
|
|
115
|
+
uniqueTargetNodes,
|
|
116
|
+
totalNodes
|
|
117
|
+
};
|
|
118
|
+
}
|
|
78
119
|
/**
|
|
79
120
|
* Add relationship to index - O(1) amortized
|
|
80
121
|
*/
|
|
@@ -98,6 +139,9 @@ export class GraphAdjacencyIndex {
|
|
|
98
139
|
// Cache immediately for hot data
|
|
99
140
|
await this.cacheIndexEntry(verb.sourceId, 'source');
|
|
100
141
|
await this.cacheIndexEntry(verb.targetId, 'target');
|
|
142
|
+
// Update type-specific counts atomically
|
|
143
|
+
const verbType = verb.type || 'unknown';
|
|
144
|
+
this.relationshipCountsByType.set(verbType, (this.relationshipCountsByType.get(verbType) || 0) + 1);
|
|
101
145
|
const elapsed = performance.now() - startTime;
|
|
102
146
|
this.totalRelationshipsIndexed++;
|
|
103
147
|
// Performance assertion
|
|
@@ -115,6 +159,15 @@ export class GraphAdjacencyIndex {
|
|
|
115
159
|
const startTime = performance.now();
|
|
116
160
|
// Remove from verb cache
|
|
117
161
|
this.verbIndex.delete(verbId);
|
|
162
|
+
// Update type-specific counts atomically
|
|
163
|
+
const verbType = verb.type || 'unknown';
|
|
164
|
+
const currentCount = this.relationshipCountsByType.get(verbType) || 0;
|
|
165
|
+
if (currentCount > 1) {
|
|
166
|
+
this.relationshipCountsByType.set(verbType, currentCount - 1);
|
|
167
|
+
}
|
|
168
|
+
else {
|
|
169
|
+
this.relationshipCountsByType.delete(verbType);
|
|
170
|
+
}
|
|
118
171
|
// Remove from source index
|
|
119
172
|
const sourceNeighbors = this.sourceIndex.get(verb.sourceId);
|
|
120
173
|
if (sourceNeighbors) {
|
package/dist/utils/logger.d.ts
CHANGED
package/dist/utils/logger.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { isProductionEnvironment, getLogLevel } from './environment.js';
|
|
7
7
|
export var LogLevel;
|
|
8
8
|
(function (LogLevel) {
|
|
9
|
+
LogLevel[LogLevel["SILENT"] = -1] = "SILENT";
|
|
9
10
|
LogLevel[LogLevel["ERROR"] = 0] = "ERROR";
|
|
10
11
|
LogLevel[LogLevel["WARN"] = 1] = "WARN";
|
|
11
12
|
LogLevel[LogLevel["INFO"] = 2] = "INFO";
|
|
@@ -45,7 +46,7 @@ class Logger {
|
|
|
45
46
|
// Convert environment log level to Logger LogLevel
|
|
46
47
|
switch (envLogLevel) {
|
|
47
48
|
case 'silent':
|
|
48
|
-
this.config.level =
|
|
49
|
+
this.config.level = LogLevel.SILENT;
|
|
49
50
|
break;
|
|
50
51
|
case 'error':
|
|
51
52
|
this.config.level = LogLevel.ERROR;
|
|
@@ -81,6 +82,10 @@ class Logger {
|
|
|
81
82
|
this.config = { ...this.config, ...config };
|
|
82
83
|
}
|
|
83
84
|
shouldLog(level, module) {
|
|
85
|
+
// Silent mode - never log anything
|
|
86
|
+
if (this.config.level === LogLevel.SILENT) {
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
84
89
|
// Check module-specific level first
|
|
85
90
|
if (this.config.modules && this.config.modules[module] !== undefined) {
|
|
86
91
|
return level <= this.config.modules[module];
|
|
@@ -224,7 +224,24 @@ export declare class MetadataIndexManager {
|
|
|
224
224
|
*/
|
|
225
225
|
private loadSortedIndex;
|
|
226
226
|
/**
|
|
227
|
-
* Get
|
|
227
|
+
* Get count of entities by type - O(1) operation using existing tracking
|
|
228
|
+
* This exposes the production-ready counting that's already maintained
|
|
229
|
+
*/
|
|
230
|
+
getEntityCountByType(type: string): number;
|
|
231
|
+
/**
|
|
232
|
+
* Get total count of all entities - O(1) operation
|
|
233
|
+
*/
|
|
234
|
+
getTotalEntityCount(): number;
|
|
235
|
+
/**
|
|
236
|
+
* Get all entity types and their counts - O(1) operation
|
|
237
|
+
*/
|
|
238
|
+
getAllEntityCounts(): Map<string, number>;
|
|
239
|
+
/**
|
|
240
|
+
* Get count of entities matching field-value criteria - O(1) lookup from existing indexes
|
|
241
|
+
*/
|
|
242
|
+
getCountForCriteria(field: string, value: any): Promise<number>;
|
|
243
|
+
/**
|
|
244
|
+
* Get index statistics with enhanced counting information
|
|
228
245
|
*/
|
|
229
246
|
getStats(): Promise<MetadataIndexStats>;
|
|
230
247
|
/**
|
|
@@ -1209,7 +1209,45 @@ export class MetadataIndexManager {
|
|
|
1209
1209
|
return cached;
|
|
1210
1210
|
}
|
|
1211
1211
|
/**
|
|
1212
|
-
* Get
|
|
1212
|
+
* Get count of entities by type - O(1) operation using existing tracking
|
|
1213
|
+
* This exposes the production-ready counting that's already maintained
|
|
1214
|
+
*/
|
|
1215
|
+
getEntityCountByType(type) {
|
|
1216
|
+
return this.totalEntitiesByType.get(type) || 0;
|
|
1217
|
+
}
|
|
1218
|
+
/**
|
|
1219
|
+
* Get total count of all entities - O(1) operation
|
|
1220
|
+
*/
|
|
1221
|
+
getTotalEntityCount() {
|
|
1222
|
+
let total = 0;
|
|
1223
|
+
for (const count of this.totalEntitiesByType.values()) {
|
|
1224
|
+
total += count;
|
|
1225
|
+
}
|
|
1226
|
+
return total;
|
|
1227
|
+
}
|
|
1228
|
+
/**
|
|
1229
|
+
* Get all entity types and their counts - O(1) operation
|
|
1230
|
+
*/
|
|
1231
|
+
getAllEntityCounts() {
|
|
1232
|
+
return new Map(this.totalEntitiesByType);
|
|
1233
|
+
}
|
|
1234
|
+
/**
|
|
1235
|
+
* Get count of entities matching field-value criteria - O(1) lookup from existing indexes
|
|
1236
|
+
*/
|
|
1237
|
+
async getCountForCriteria(field, value) {
|
|
1238
|
+
const key = this.getIndexKey(field, value);
|
|
1239
|
+
let entry = this.indexCache.get(key);
|
|
1240
|
+
if (!entry) {
|
|
1241
|
+
const loadedEntry = await this.loadIndexEntry(key);
|
|
1242
|
+
if (loadedEntry) {
|
|
1243
|
+
entry = loadedEntry;
|
|
1244
|
+
this.indexCache.set(key, entry);
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
return entry ? entry.ids.size : 0;
|
|
1248
|
+
}
|
|
1249
|
+
/**
|
|
1250
|
+
* Get index statistics with enhanced counting information
|
|
1213
1251
|
*/
|
|
1214
1252
|
async getStats() {
|
|
1215
1253
|
const fields = new Set();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soulcraft/brainy",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.3.0",
|
|
4
4
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.js",
|