@soulcraft/brainy 5.3.6 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +110 -0
- package/README.md +4 -3
- package/dist/augmentations/display/fieldPatterns.js +3 -3
- package/dist/augmentations/display/intelligentComputation.js +0 -2
- package/dist/augmentations/typeMatching/brainyTypes.js +6 -8
- package/dist/brainy.d.ts +61 -0
- package/dist/brainy.js +180 -24
- package/dist/cortex/neuralImport.js +0 -1
- package/dist/importers/SmartExcelImporter.js +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/neural/embeddedKeywordEmbeddings.d.ts +1 -1
- package/dist/neural/embeddedKeywordEmbeddings.js +56 -56
- package/dist/neural/embeddedTypeEmbeddings.d.ts +3 -3
- package/dist/neural/embeddedTypeEmbeddings.js +14 -14
- package/dist/neural/entityExtractor.js +2 -2
- package/dist/neural/relationshipConfidence.js +1 -1
- package/dist/neural/signals/VerbContextSignal.js +6 -6
- package/dist/neural/signals/VerbExactMatchSignal.js +9 -9
- package/dist/neural/signals/VerbPatternSignal.js +5 -5
- package/dist/query/typeAwareQueryPlanner.js +2 -3
- package/dist/storage/adapters/azureBlobStorage.d.ts +13 -64
- package/dist/storage/adapters/azureBlobStorage.js +78 -388
- package/dist/storage/adapters/fileSystemStorage.d.ts +12 -78
- package/dist/storage/adapters/fileSystemStorage.js +49 -395
- package/dist/storage/adapters/gcsStorage.d.ts +13 -134
- package/dist/storage/adapters/gcsStorage.js +79 -557
- package/dist/storage/adapters/historicalStorageAdapter.d.ts +181 -0
- package/dist/storage/adapters/historicalStorageAdapter.js +332 -0
- package/dist/storage/adapters/memoryStorage.d.ts +4 -113
- package/dist/storage/adapters/memoryStorage.js +34 -471
- package/dist/storage/adapters/opfsStorage.d.ts +14 -127
- package/dist/storage/adapters/opfsStorage.js +44 -693
- package/dist/storage/adapters/r2Storage.d.ts +8 -41
- package/dist/storage/adapters/r2Storage.js +49 -237
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +13 -111
- package/dist/storage/adapters/s3CompatibleStorage.js +77 -596
- package/dist/storage/baseStorage.d.ts +78 -38
- package/dist/storage/baseStorage.js +692 -23
- package/dist/storage/cow/BlobStorage.d.ts +2 -2
- package/dist/storage/cow/BlobStorage.js +4 -4
- package/dist/storage/storageFactory.d.ts +2 -3
- package/dist/storage/storageFactory.js +114 -66
- package/dist/types/graphTypes.d.ts +588 -230
- package/dist/types/graphTypes.js +683 -248
- package/dist/types/typeMigration.d.ts +95 -0
- package/dist/types/typeMigration.js +141 -0
- package/dist/utils/intelligentTypeMapper.js +2 -2
- package/dist/utils/metadataIndex.js +6 -6
- package/dist/vfs/types.d.ts +6 -2
- package/package.json +2 -2
|
@@ -234,11 +234,11 @@ export class NeuralEntityExtractor {
|
|
|
234
234
|
}
|
|
235
235
|
// Hashtag
|
|
236
236
|
if (text.startsWith('#')) {
|
|
237
|
-
return { type: NounType.
|
|
237
|
+
return { type: NounType.Concept, confidence: 0.8 };
|
|
238
238
|
}
|
|
239
239
|
// Mention
|
|
240
240
|
if (text.startsWith('@')) {
|
|
241
|
-
return { type: NounType.
|
|
241
|
+
return { type: NounType.Person, confidence: 0.8 };
|
|
242
242
|
}
|
|
243
243
|
// Capitalized words (likely proper nouns)
|
|
244
244
|
if (/^[A-Z]/.test(text)) {
|
|
@@ -106,7 +106,7 @@ export class RelationshipConfidenceScorer {
|
|
|
106
106
|
[VerbType.Contains]: ['contains', 'includes', 'has', 'holds'],
|
|
107
107
|
[VerbType.Requires]: ['requires', 'needs', 'depends on', 'relies on'],
|
|
108
108
|
[VerbType.Uses]: ['uses', 'utilizes', 'employs', 'applies'],
|
|
109
|
-
[VerbType.
|
|
109
|
+
[VerbType.ReportsTo]: ['manages', 'oversees', 'supervises', 'controls'],
|
|
110
110
|
[VerbType.Causes]: ['influences', 'affects', 'impacts', 'shapes', 'causes'],
|
|
111
111
|
[VerbType.DependsOn]: ['depends on', 'relies on', 'based on'],
|
|
112
112
|
[VerbType.Modifies]: ['modifies', 'changes', 'alters', 'updates'],
|
|
@@ -105,7 +105,7 @@ export class VerbContextSignal {
|
|
|
105
105
|
{
|
|
106
106
|
subjectType: NounType.Document,
|
|
107
107
|
objectType: NounType.Person,
|
|
108
|
-
verbType: VerbType.
|
|
108
|
+
verbType: VerbType.Creates,
|
|
109
109
|
confidence: 0.80,
|
|
110
110
|
description: 'Document created by Person'
|
|
111
111
|
},
|
|
@@ -173,7 +173,7 @@ export class VerbContextSignal {
|
|
|
173
173
|
{
|
|
174
174
|
subjectType: NounType.Product,
|
|
175
175
|
objectType: NounType.Organization,
|
|
176
|
-
verbType: VerbType.
|
|
176
|
+
verbType: VerbType.Creates,
|
|
177
177
|
confidence: 0.75,
|
|
178
178
|
description: 'Product created by Organization'
|
|
179
179
|
},
|
|
@@ -188,7 +188,7 @@ export class VerbContextSignal {
|
|
|
188
188
|
{
|
|
189
189
|
subjectType: NounType.Product,
|
|
190
190
|
objectType: NounType.Person,
|
|
191
|
-
verbType: VerbType.
|
|
191
|
+
verbType: VerbType.Creates,
|
|
192
192
|
confidence: 0.75,
|
|
193
193
|
description: 'Product created by Person'
|
|
194
194
|
},
|
|
@@ -196,7 +196,7 @@ export class VerbContextSignal {
|
|
|
196
196
|
{
|
|
197
197
|
subjectType: NounType.Event,
|
|
198
198
|
objectType: NounType.Person,
|
|
199
|
-
verbType: VerbType.
|
|
199
|
+
verbType: VerbType.Creates,
|
|
200
200
|
confidence: 0.70,
|
|
201
201
|
description: 'Event created by Person'
|
|
202
202
|
},
|
|
@@ -220,7 +220,7 @@ export class VerbContextSignal {
|
|
|
220
220
|
{
|
|
221
221
|
subjectType: NounType.Project,
|
|
222
222
|
objectType: NounType.Organization,
|
|
223
|
-
verbType: VerbType.
|
|
223
|
+
verbType: VerbType.Owns,
|
|
224
224
|
confidence: 0.75,
|
|
225
225
|
description: 'Project belongs to Organization'
|
|
226
226
|
},
|
|
@@ -228,7 +228,7 @@ export class VerbContextSignal {
|
|
|
228
228
|
{
|
|
229
229
|
subjectType: NounType.Project,
|
|
230
230
|
objectType: NounType.Person,
|
|
231
|
-
verbType: VerbType.
|
|
231
|
+
verbType: VerbType.Creates,
|
|
232
232
|
confidence: 0.70,
|
|
233
233
|
description: 'Project created by Person'
|
|
234
234
|
},
|
|
@@ -190,14 +190,14 @@ export class VerbExactMatchSignal {
|
|
|
190
190
|
// Common relationship phrases with their VerbTypes
|
|
191
191
|
const phrases = [
|
|
192
192
|
// Creation relationships
|
|
193
|
-
{ pattern: /created?\s+by/i, type: VerbType.
|
|
194
|
-
{ pattern: /authored?\s+by/i, type: VerbType.
|
|
195
|
-
{ pattern: /written\s+by/i, type: VerbType.
|
|
196
|
-
{ pattern: /developed\s+by/i, type: VerbType.
|
|
193
|
+
{ pattern: /created?\s+by/i, type: VerbType.Creates, confidence: 0.95 },
|
|
194
|
+
{ pattern: /authored?\s+by/i, type: VerbType.Creates, confidence: 0.95 },
|
|
195
|
+
{ pattern: /written\s+by/i, type: VerbType.Creates, confidence: 0.95 },
|
|
196
|
+
{ pattern: /developed\s+by/i, type: VerbType.Creates, confidence: 0.90 },
|
|
197
197
|
{ pattern: /built\s+by/i, type: VerbType.Creates, confidence: 0.85 },
|
|
198
198
|
// Ownership relationships
|
|
199
199
|
{ pattern: /owned\s+by/i, type: VerbType.Owns, confidence: 0.95 },
|
|
200
|
-
{ pattern: /belongs\s+to/i, type: VerbType.
|
|
200
|
+
{ pattern: /belongs\s+to/i, type: VerbType.Owns, confidence: 0.95 },
|
|
201
201
|
{ pattern: /attributed\s+to/i, type: VerbType.AttributedTo, confidence: 0.95 },
|
|
202
202
|
// Part/Whole relationships
|
|
203
203
|
{ pattern: /part\s+of/i, type: VerbType.PartOf, confidence: 0.95 },
|
|
@@ -213,17 +213,17 @@ export class VerbExactMatchSignal {
|
|
|
213
213
|
{ pattern: /employed\s+by/i, type: VerbType.WorksWith, confidence: 0.90 },
|
|
214
214
|
// Reporting relationships
|
|
215
215
|
{ pattern: /reports?\s+to/i, type: VerbType.ReportsTo, confidence: 0.95 },
|
|
216
|
-
{ pattern: /manages/i, type: VerbType.
|
|
217
|
-
{ pattern: /supervises/i, type: VerbType.
|
|
216
|
+
{ pattern: /manages/i, type: VerbType.ReportsTo, confidence: 0.85 },
|
|
217
|
+
{ pattern: /supervises/i, type: VerbType.ReportsTo, confidence: 0.95 },
|
|
218
218
|
// Reference relationships
|
|
219
219
|
{ pattern: /references/i, type: VerbType.References, confidence: 0.90 },
|
|
220
220
|
{ pattern: /cites/i, type: VerbType.References, confidence: 0.90 },
|
|
221
221
|
{ pattern: /mentions/i, type: VerbType.References, confidence: 0.85 },
|
|
222
222
|
// Temporal relationships
|
|
223
223
|
{ pattern: /precedes/i, type: VerbType.Precedes, confidence: 0.90 },
|
|
224
|
-
{ pattern: /follows/i, type: VerbType.
|
|
224
|
+
{ pattern: /follows/i, type: VerbType.Precedes, confidence: 0.90 },
|
|
225
225
|
{ pattern: /before/i, type: VerbType.Precedes, confidence: 0.75 },
|
|
226
|
-
{ pattern: /after/i, type: VerbType.
|
|
226
|
+
{ pattern: /after/i, type: VerbType.Precedes, confidence: 0.75 },
|
|
227
227
|
// Causal relationships
|
|
228
228
|
{ pattern: /causes/i, type: VerbType.Causes, confidence: 0.90 },
|
|
229
229
|
{ pattern: /requires/i, type: VerbType.Requires, confidence: 0.90 },
|
|
@@ -53,7 +53,7 @@ export class VerbPatternSignal {
|
|
|
53
53
|
// ========== Creation & Authorship ==========
|
|
54
54
|
{
|
|
55
55
|
regex: /\b(?:created?|made|built|developed|designed|wrote|authored|composed)\s+(?:by|from)\b/i,
|
|
56
|
-
type: VerbType.
|
|
56
|
+
type: VerbType.Creates,
|
|
57
57
|
confidence: 0.90,
|
|
58
58
|
description: 'Creation with agent (passive)'
|
|
59
59
|
},
|
|
@@ -84,7 +84,7 @@ export class VerbPatternSignal {
|
|
|
84
84
|
},
|
|
85
85
|
{
|
|
86
86
|
regex: /\bbelongs?\s+to\b/i,
|
|
87
|
-
type: VerbType.
|
|
87
|
+
type: VerbType.Owns,
|
|
88
88
|
confidence: 0.95,
|
|
89
89
|
description: 'Belonging relationship'
|
|
90
90
|
},
|
|
@@ -141,7 +141,7 @@ export class VerbPatternSignal {
|
|
|
141
141
|
},
|
|
142
142
|
{
|
|
143
143
|
regex: /\b(?:manages?|supervises?|oversees?)\b/i,
|
|
144
|
-
type: VerbType.
|
|
144
|
+
type: VerbType.ReportsTo,
|
|
145
145
|
confidence: 0.85,
|
|
146
146
|
description: 'Management relationship'
|
|
147
147
|
},
|
|
@@ -198,7 +198,7 @@ export class VerbPatternSignal {
|
|
|
198
198
|
},
|
|
199
199
|
{
|
|
200
200
|
regex: /\b(?:succeeds?|follows?|comes?\s+after|happens?\s+after)\b/i,
|
|
201
|
-
type: VerbType.
|
|
201
|
+
type: VerbType.Precedes,
|
|
202
202
|
confidence: 0.85,
|
|
203
203
|
description: 'Temporal succession'
|
|
204
204
|
},
|
|
@@ -210,7 +210,7 @@ export class VerbPatternSignal {
|
|
|
210
210
|
},
|
|
211
211
|
{
|
|
212
212
|
regex: /\bafter\b/i,
|
|
213
|
-
type: VerbType.
|
|
213
|
+
type: VerbType.Precedes,
|
|
214
214
|
confidence: 0.70,
|
|
215
215
|
description: 'After (temporal)'
|
|
216
216
|
},
|
|
@@ -145,18 +145,17 @@ export class TypeAwareQueryPlanner {
|
|
|
145
145
|
NounType.Media,
|
|
146
146
|
NounType.File,
|
|
147
147
|
NounType.Message,
|
|
148
|
-
NounType.Content,
|
|
149
148
|
NounType.Collection,
|
|
150
149
|
NounType.Dataset,
|
|
151
150
|
NounType.Product,
|
|
152
151
|
NounType.Service,
|
|
153
|
-
NounType.
|
|
152
|
+
NounType.Person,
|
|
154
153
|
NounType.Task,
|
|
155
154
|
NounType.Project,
|
|
156
155
|
NounType.Process,
|
|
157
156
|
NounType.State,
|
|
158
157
|
NounType.Role,
|
|
159
|
-
NounType.
|
|
158
|
+
NounType.Concept,
|
|
160
159
|
NounType.Language,
|
|
161
160
|
NounType.Currency,
|
|
162
161
|
NounType.Measurement,
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*
|
|
12
12
|
* v4.0.0: Fully compatible with metadata/vector separation architecture
|
|
13
13
|
*/
|
|
14
|
-
import { HNSWNoun, HNSWVerb,
|
|
14
|
+
import { HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
|
15
15
|
import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
|
|
16
16
|
type HNSWNode = HNSWNoun;
|
|
17
17
|
type Edge = HNSWVerb;
|
|
@@ -24,6 +24,12 @@ type Edge = HNSWVerb;
|
|
|
24
24
|
* 2. Connection String - if connectionString provided
|
|
25
25
|
* 3. Storage Account Key - if accountName + accountKey provided
|
|
26
26
|
* 4. SAS Token - if accountName + sasToken provided
|
|
27
|
+
*
|
|
28
|
+
* v5.4.0: Type-aware storage now built into BaseStorage
|
|
29
|
+
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
|
30
|
+
* - Removed pagination overrides
|
|
31
|
+
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
|
32
|
+
* - All operations now use type-first paths: entities/nouns/{type}/vectors/{shard}/{id}.json
|
|
27
33
|
*/
|
|
28
34
|
export declare class AzureBlobStorage extends BaseStorage {
|
|
29
35
|
private blobServiceClient;
|
|
@@ -53,6 +59,7 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
53
59
|
private nounCacheManager;
|
|
54
60
|
private verbCacheManager;
|
|
55
61
|
private logger;
|
|
62
|
+
private hnswLocks;
|
|
56
63
|
/**
|
|
57
64
|
* Initialize the storage adapter
|
|
58
65
|
* @param options Configuration options for Azure Blob Storage
|
|
@@ -146,10 +153,6 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
146
153
|
* Flush verb buffer to Azure
|
|
147
154
|
*/
|
|
148
155
|
private flushVerbBuffer;
|
|
149
|
-
/**
|
|
150
|
-
* Save a noun to storage (internal implementation)
|
|
151
|
-
*/
|
|
152
|
-
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
|
153
156
|
/**
|
|
154
157
|
* Save a node to storage
|
|
155
158
|
*/
|
|
@@ -158,20 +161,10 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
158
161
|
* Save a node directly to Azure (bypass buffer)
|
|
159
162
|
*/
|
|
160
163
|
private saveNodeDirect;
|
|
161
|
-
/**
|
|
162
|
-
* Get a noun from storage (internal implementation)
|
|
163
|
-
* v4.0.0: Returns ONLY vector data (no metadata field)
|
|
164
|
-
* Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
|
|
165
|
-
*/
|
|
166
|
-
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
|
167
164
|
/**
|
|
168
165
|
* Get a node from storage
|
|
169
166
|
*/
|
|
170
167
|
protected getNode(id: string): Promise<HNSWNode | null>;
|
|
171
|
-
/**
|
|
172
|
-
* Delete a noun from storage (internal implementation)
|
|
173
|
-
*/
|
|
174
|
-
protected deleteNoun_internal(id: string): Promise<void>;
|
|
175
168
|
/**
|
|
176
169
|
* Write an object to a specific path in Azure
|
|
177
170
|
* Primitive operation required by base class
|
|
@@ -222,10 +215,6 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
222
215
|
* Helper: Convert Azure stream to buffer
|
|
223
216
|
*/
|
|
224
217
|
private streamToBuffer;
|
|
225
|
-
/**
|
|
226
|
-
* Save a verb to storage (internal implementation)
|
|
227
|
-
*/
|
|
228
|
-
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
|
229
218
|
/**
|
|
230
219
|
* Save an edge to storage
|
|
231
220
|
*/
|
|
@@ -234,55 +223,10 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
234
223
|
* Save an edge directly to Azure (bypass buffer)
|
|
235
224
|
*/
|
|
236
225
|
private saveEdgeDirect;
|
|
237
|
-
/**
|
|
238
|
-
* Get a verb from storage (internal implementation)
|
|
239
|
-
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
240
|
-
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
241
|
-
*/
|
|
242
|
-
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
|
243
226
|
/**
|
|
244
227
|
* Get an edge from storage
|
|
245
228
|
*/
|
|
246
229
|
protected getEdge(id: string): Promise<Edge | null>;
|
|
247
|
-
/**
|
|
248
|
-
* Delete a verb from storage (internal implementation)
|
|
249
|
-
*/
|
|
250
|
-
protected deleteVerb_internal(id: string): Promise<void>;
|
|
251
|
-
/**
|
|
252
|
-
* Get nouns with pagination
|
|
253
|
-
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
|
|
254
|
-
* Iterates through all UUID-based shards (00-ff) for consistent pagination
|
|
255
|
-
*/
|
|
256
|
-
getNounsWithPagination(options?: {
|
|
257
|
-
limit?: number;
|
|
258
|
-
cursor?: string;
|
|
259
|
-
filter?: {
|
|
260
|
-
nounType?: string | string[];
|
|
261
|
-
service?: string | string[];
|
|
262
|
-
metadata?: Record<string, any>;
|
|
263
|
-
};
|
|
264
|
-
}): Promise<{
|
|
265
|
-
items: HNSWNounWithMetadata[];
|
|
266
|
-
totalCount?: number;
|
|
267
|
-
hasMore: boolean;
|
|
268
|
-
nextCursor?: string;
|
|
269
|
-
}>;
|
|
270
|
-
/**
|
|
271
|
-
* Get nouns by noun type (internal implementation)
|
|
272
|
-
*/
|
|
273
|
-
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
|
274
|
-
/**
|
|
275
|
-
* Get verbs by source ID (internal implementation)
|
|
276
|
-
*/
|
|
277
|
-
protected getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
278
|
-
/**
|
|
279
|
-
* Get verbs by target ID (internal implementation)
|
|
280
|
-
*/
|
|
281
|
-
protected getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
282
|
-
/**
|
|
283
|
-
* Get verbs by type (internal implementation)
|
|
284
|
-
*/
|
|
285
|
-
protected getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]>;
|
|
286
230
|
/**
|
|
287
231
|
* Clear all data from storage
|
|
288
232
|
*/
|
|
@@ -318,10 +262,14 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
318
262
|
protected persistCounts(): Promise<void>;
|
|
319
263
|
/**
|
|
320
264
|
* Get a noun's vector for HNSW rebuild
|
|
265
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
321
266
|
*/
|
|
322
267
|
getNounVector(id: string): Promise<number[] | null>;
|
|
323
268
|
/**
|
|
324
269
|
* Save HNSW graph data for a noun
|
|
270
|
+
*
|
|
271
|
+
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
|
272
|
+
* CRITICAL: Uses mutex locking to prevent read-modify-write races
|
|
325
273
|
*/
|
|
326
274
|
saveHNSWData(nounId: string, hnswData: {
|
|
327
275
|
level: number;
|
|
@@ -329,6 +277,7 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
329
277
|
}): Promise<void>;
|
|
330
278
|
/**
|
|
331
279
|
* Get HNSW graph data for a noun
|
|
280
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
332
281
|
*/
|
|
333
282
|
getHNSWData(nounId: string): Promise<{
|
|
334
283
|
level: number;
|