@soulcraft/brainy 4.1.2 β 4.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +126 -7
- package/dist/brainy.d.ts +105 -18
- package/dist/brainy.js +134 -34
- package/dist/import/ImportCoordinator.d.ts +53 -1
- package/dist/import/ImportCoordinator.js +98 -0
- package/dist/neural/improvedNeuralAPI.js +5 -5
- package/dist/storage/baseStorage.js +5 -3
- package/dist/types/brainy.types.d.ts +69 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,37 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
|
4
4
|
|
|
5
|
+
### [4.1.4](https://github.com/soulcraftlabs/brainy/compare/v4.1.3...v4.1.4) (2025-10-21)
|
|
6
|
+
|
|
7
|
+
- feat: add import API validation and v4.x migration guide (a1a0576)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
### [4.1.3](https://github.com/soulcraftlabs/brainy/compare/v4.1.2...v4.1.3) (2025-10-21)
|
|
11
|
+
|
|
12
|
+
- perf: make getRelations() pagination consistent and efficient (54d819c)
|
|
13
|
+
- fix: resolve getRelations() empty array bug and add string ID shorthand (8d217f3)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
### [4.1.3](https://github.com/soulcraftlabs/brainy/compare/v4.1.2...v4.1.3) (2025-10-21)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
### π Bug Fixes
|
|
20
|
+
|
|
21
|
+
* **api**: fix getRelations() returning empty array when called without parameters
|
|
22
|
+
- Fixed critical bug where `brain.getRelations()` returned `[]` instead of all relationships
|
|
23
|
+
- Added support for retrieving all relationships with pagination (default limit: 100)
|
|
24
|
+
- Added string ID shorthand syntax: `brain.getRelations(entityId)` as alias for `brain.getRelations({ from: entityId })`
|
|
25
|
+
- **Performance**: Made pagination consistent - now ALL query patterns paginate at storage layer
|
|
26
|
+
- **Efficiency**: `getRelations({ from: id, limit: 10 })` now fetches only 10 instead of fetching ALL then slicing
|
|
27
|
+
- Fixed storage.getVerbs() offset handling - now properly converts offset to cursor for adapters
|
|
28
|
+
- Production safety: Warns when fetching >10k relationships without filters
|
|
29
|
+
- Fixed broken method calls in improvedNeuralAPI.ts (replaced non-existent `getVerbsForNoun` with `getRelations`)
|
|
30
|
+
- Fixed property access bugs: `verb.target` β `verb.to`, `verb.verb` β `verb.type`
|
|
31
|
+
- Added comprehensive integration tests (14 tests covering all query patterns)
|
|
32
|
+
- Updated JSDoc documentation with usage examples
|
|
33
|
+
- **Impact**: Resolves Workshop team bug where 524 imported relationships were inaccessible
|
|
34
|
+
- **Breaking**: None - fully backward compatible
|
|
35
|
+
|
|
5
36
|
### [4.1.2](https://github.com/soulcraftlabs/brainy/compare/v4.1.1...v4.1.2) (2025-10-21)
|
|
6
37
|
|
|
7
38
|
|
|
@@ -197,22 +228,110 @@ $ brainy import ./research-papers --extract-concepts --progress
|
|
|
197
228
|
|
|
198
229
|
### β οΈ Breaking Changes
|
|
199
230
|
|
|
200
|
-
|
|
231
|
+
#### π₯ Import API Redesign
|
|
232
|
+
|
|
233
|
+
The import API has been redesigned for clarity and better feature control. **Old v3.x option names are no longer recognized** and will throw errors.
|
|
234
|
+
|
|
235
|
+
**What Changed:**
|
|
236
|
+
|
|
237
|
+
| v3.x Option | v4.x Option | Action Required |
|
|
238
|
+
|-------------|-------------|-----------------|
|
|
239
|
+
| `extractRelationships` | `enableRelationshipInference` | **Rename option** |
|
|
240
|
+
| `autoDetect` | *(removed)* | **Delete option** (always enabled) |
|
|
241
|
+
| `createFileStructure` | `vfsPath` | **Replace** with VFS path |
|
|
242
|
+
| `excelSheets` | *(removed)* | **Delete option** (all sheets processed) |
|
|
243
|
+
| `pdfExtractTables` | *(removed)* | **Delete option** (always enabled) |
|
|
244
|
+
| - | `enableNeuralExtraction` | **Add option** (new in v4.x) |
|
|
245
|
+
| - | `enableConceptExtraction` | **Add option** (new in v4.x) |
|
|
246
|
+
| - | `preserveSource` | **Add option** (new in v4.x) |
|
|
247
|
+
|
|
248
|
+
**Why These Changes?**
|
|
249
|
+
|
|
250
|
+
1. **Clearer option names**: `enableRelationshipInference` explicitly indicates AI-powered relationship inference
|
|
251
|
+
2. **Separation of concerns**: Neural extraction, relationship inference, and VFS are now separate, explicit options
|
|
252
|
+
3. **Better defaults**: Auto-detection and AI features are enabled by default
|
|
253
|
+
4. **Reduced confusion**: Removed redundant options like `autoDetect` and format-specific options
|
|
254
|
+
|
|
255
|
+
**Migration Examples:**
|
|
256
|
+
|
|
257
|
+
<details>
|
|
258
|
+
<summary>Example 1: Basic Excel Import</summary>
|
|
259
|
+
|
|
260
|
+
```typescript
|
|
261
|
+
// v3.x (OLD - Will throw error)
|
|
262
|
+
await brain.import('./glossary.xlsx', {
|
|
263
|
+
extractRelationships: true,
|
|
264
|
+
createFileStructure: true
|
|
265
|
+
})
|
|
266
|
+
|
|
267
|
+
// v4.x (NEW - Use this)
|
|
268
|
+
await brain.import('./glossary.xlsx', {
|
|
269
|
+
enableRelationshipInference: true,
|
|
270
|
+
vfsPath: '/imports/glossary'
|
|
271
|
+
})
|
|
272
|
+
```
|
|
273
|
+
</details>
|
|
274
|
+
|
|
275
|
+
<details>
|
|
276
|
+
<summary>Example 2: Full-Featured Import</summary>
|
|
277
|
+
|
|
278
|
+
```typescript
|
|
279
|
+
// v3.x (OLD - Will throw error)
|
|
280
|
+
await brain.import('./data.xlsx', {
|
|
281
|
+
extractRelationships: true,
|
|
282
|
+
autoDetect: true,
|
|
283
|
+
createFileStructure: true
|
|
284
|
+
})
|
|
285
|
+
|
|
286
|
+
// v4.x (NEW - Use this)
|
|
287
|
+
await brain.import('./data.xlsx', {
|
|
288
|
+
enableNeuralExtraction: true, // Extract entity names
|
|
289
|
+
enableRelationshipInference: true, // Infer semantic relationships
|
|
290
|
+
enableConceptExtraction: true, // Extract entity types
|
|
291
|
+
vfsPath: '/imports/data', // VFS directory
|
|
292
|
+
preserveSource: true // Save original file
|
|
293
|
+
})
|
|
294
|
+
```
|
|
295
|
+
</details>
|
|
296
|
+
|
|
297
|
+
**Error Messages:**
|
|
201
298
|
|
|
202
|
-
|
|
299
|
+
If you use old v3.x options, you'll get a clear error message:
|
|
300
|
+
|
|
301
|
+
```
|
|
302
|
+
β Invalid import options detected (Brainy v4.x breaking changes)
|
|
303
|
+
|
|
304
|
+
The following v3.x options are no longer supported:
|
|
305
|
+
|
|
306
|
+
β extractRelationships
|
|
307
|
+
β Use: enableRelationshipInference
|
|
308
|
+
β Why: Option renamed for clarity in v4.x
|
|
309
|
+
|
|
310
|
+
π Migration Guide: https://brainy.dev/docs/guides/migrating-to-v4
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
**Other v4.0.0 Features (Non-Breaking):**
|
|
314
|
+
|
|
315
|
+
All other v4.0.0 features are:
|
|
203
316
|
- β
Opt-in (lifecycle, compression, batch operations)
|
|
204
317
|
- β
Additive (new CLI commands, new methods)
|
|
205
318
|
- β
Non-breaking (existing code continues to work)
|
|
206
319
|
|
|
207
320
|
### π Migration
|
|
208
321
|
|
|
209
|
-
**
|
|
322
|
+
**Import API migration required** if you use `brain.import()` with the old v3.x option names.
|
|
210
323
|
|
|
211
|
-
|
|
324
|
+
#### Required Changes:
|
|
212
325
|
1. Update to v4.0.0: `npm install @soulcraft/brainy@4.0.0`
|
|
213
|
-
2.
|
|
214
|
-
3.
|
|
215
|
-
|
|
326
|
+
2. Update import calls to use new option names (see table above)
|
|
327
|
+
3. Test your imports - you'll get clear error messages if you use old options
|
|
328
|
+
|
|
329
|
+
#### Optional Enhancements:
|
|
330
|
+
- Enable lifecycle policies: `brainy storage lifecycle set`
|
|
331
|
+
- Use batch operations: `brainy storage batch-delete entities.txt`
|
|
332
|
+
- See full migration guide: `docs/guides/migrating-to-v4.md`
|
|
333
|
+
|
|
334
|
+
**Complete Migration Guide:** [docs/guides/migrating-to-v4.md](./docs/guides/migrating-to-v4.md)
|
|
216
335
|
|
|
217
336
|
### π What This Means
|
|
218
337
|
|
package/dist/brainy.d.ts
CHANGED
|
@@ -314,9 +314,38 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
314
314
|
*/
|
|
315
315
|
unrelate(id: string): Promise<void>;
|
|
316
316
|
/**
|
|
317
|
-
* Get relationships
|
|
317
|
+
* Get relationships between entities
|
|
318
|
+
*
|
|
319
|
+
* Supports multiple query patterns:
|
|
320
|
+
* - No parameters: Returns all relationships (paginated, default limit: 100)
|
|
321
|
+
* - String ID: Returns relationships from that entity (shorthand for { from: id })
|
|
322
|
+
* - Parameters object: Fine-grained filtering and pagination
|
|
323
|
+
*
|
|
324
|
+
* @param paramsOrId - Optional string ID or parameters object
|
|
325
|
+
* @returns Promise resolving to array of relationships
|
|
326
|
+
*
|
|
327
|
+
* @example
|
|
328
|
+
* ```typescript
|
|
329
|
+
* // Get all relationships (first 100)
|
|
330
|
+
* const all = await brain.getRelations()
|
|
331
|
+
*
|
|
332
|
+
* // Get relationships from specific entity (shorthand syntax)
|
|
333
|
+
* const fromEntity = await brain.getRelations(entityId)
|
|
334
|
+
*
|
|
335
|
+
* // Get relationships with filters
|
|
336
|
+
* const filtered = await brain.getRelations({
|
|
337
|
+
* type: VerbType.FriendOf,
|
|
338
|
+
* limit: 50
|
|
339
|
+
* })
|
|
340
|
+
*
|
|
341
|
+
* // Pagination
|
|
342
|
+
* const page2 = await brain.getRelations({ offset: 100, limit: 100 })
|
|
343
|
+
* ```
|
|
344
|
+
*
|
|
345
|
+
* @since v4.1.3 - Fixed bug where calling without parameters returned empty array
|
|
346
|
+
* @since v4.1.3 - Added string ID shorthand syntax: getRelations(id)
|
|
318
347
|
*/
|
|
319
|
-
getRelations(
|
|
348
|
+
getRelations(paramsOrId?: string | GetRelationsParams): Promise<Relation<T>[]>;
|
|
320
349
|
/**
|
|
321
350
|
* Unified find method - supports natural language and structured queries
|
|
322
351
|
* Implements Triple Intelligence with parallel search optimization
|
|
@@ -657,33 +686,91 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
657
686
|
limit?: number;
|
|
658
687
|
}): Promise<string[]>;
|
|
659
688
|
/**
|
|
660
|
-
* Import files with
|
|
689
|
+
* Import files with intelligent extraction and dual storage (VFS + Knowledge Graph)
|
|
661
690
|
*
|
|
662
691
|
* Unified import system that:
|
|
663
692
|
* - Auto-detects format (Excel, PDF, CSV, JSON, Markdown)
|
|
664
|
-
* - Extracts entities
|
|
693
|
+
* - Extracts entities with AI-powered name/type detection
|
|
694
|
+
* - Infers semantic relationships from context
|
|
665
695
|
* - Stores in both VFS (organized files) and Knowledge Graph (connected entities)
|
|
666
696
|
* - Links VFS files to graph entities
|
|
667
697
|
*
|
|
668
|
-
* @
|
|
669
|
-
* // Import from file path
|
|
670
|
-
* const result = await brain.import('/path/to/file.xlsx')
|
|
698
|
+
* @since 4.0.0
|
|
671
699
|
*
|
|
672
|
-
* @example
|
|
673
|
-
*
|
|
700
|
+
* @example Quick Start (All AI features enabled by default)
|
|
701
|
+
* ```typescript
|
|
702
|
+
* const result = await brain.import('./glossary.xlsx')
|
|
703
|
+
* // Auto-detects format, extracts entities, infers relationships
|
|
704
|
+
* ```
|
|
705
|
+
*
|
|
706
|
+
* @example Full-Featured Import (v4.x)
|
|
707
|
+
* ```typescript
|
|
708
|
+
* const result = await brain.import('./data.xlsx', {
|
|
709
|
+
* // AI features
|
|
710
|
+
* enableNeuralExtraction: true, // Extract entity names/metadata
|
|
711
|
+
* enableRelationshipInference: true, // Detect semantic relationships
|
|
712
|
+
* enableConceptExtraction: true, // Extract types/concepts
|
|
713
|
+
*
|
|
714
|
+
* // VFS features
|
|
715
|
+
* vfsPath: '/imports/my-data', // Store in VFS directory
|
|
716
|
+
* groupBy: 'type', // Organize by entity type
|
|
717
|
+
* preserveSource: true, // Keep original file
|
|
718
|
+
*
|
|
719
|
+
* // Progress tracking
|
|
720
|
+
* onProgress: (p) => console.log(p.message)
|
|
721
|
+
* })
|
|
722
|
+
* ```
|
|
723
|
+
*
|
|
724
|
+
* @example Performance Tuning (Large Files)
|
|
725
|
+
* ```typescript
|
|
726
|
+
* const result = await brain.import('./huge-file.csv', {
|
|
727
|
+
* enableDeduplication: false, // Skip dedup for speed
|
|
728
|
+
* confidenceThreshold: 0.8, // Higher threshold = fewer entities
|
|
729
|
+
* onProgress: (p) => console.log(`${p.processed}/${p.total}`)
|
|
730
|
+
* })
|
|
731
|
+
* ```
|
|
732
|
+
*
|
|
733
|
+
* @example Import from Buffer or Object
|
|
734
|
+
* ```typescript
|
|
735
|
+
* // From buffer
|
|
674
736
|
* const result = await brain.import(buffer, { format: 'pdf' })
|
|
675
737
|
*
|
|
676
|
-
*
|
|
677
|
-
* // Import JSON object
|
|
738
|
+
* // From object
|
|
678
739
|
* const result = await brain.import({ entities: [...] })
|
|
740
|
+
* ```
|
|
679
741
|
*
|
|
680
|
-
* @
|
|
681
|
-
*
|
|
682
|
-
*
|
|
683
|
-
*
|
|
684
|
-
*
|
|
685
|
-
*
|
|
686
|
-
*
|
|
742
|
+
* @throws {Error} If invalid options are provided (v4.x breaking changes)
|
|
743
|
+
*
|
|
744
|
+
* @see {@link https://brainy.dev/docs/api/import API Documentation}
|
|
745
|
+
* @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide}
|
|
746
|
+
*
|
|
747
|
+
* @remarks
|
|
748
|
+
* **β οΈ Breaking Changes from v3.x:**
|
|
749
|
+
*
|
|
750
|
+
* The import API was redesigned in v4.0.0 for clarity and better feature control.
|
|
751
|
+
* Old v3.x option names are **no longer recognized** and will throw errors.
|
|
752
|
+
*
|
|
753
|
+
* **Option Changes:**
|
|
754
|
+
* - β `extractRelationships` β β
`enableRelationshipInference`
|
|
755
|
+
* - β `createFileStructure` β β
`vfsPath: '/your/path'`
|
|
756
|
+
* - β `autoDetect` β β
*(removed - always enabled)*
|
|
757
|
+
* - β `excelSheets` β β
*(removed - all sheets processed)*
|
|
758
|
+
* - β `pdfExtractTables` β β
*(removed - always enabled)*
|
|
759
|
+
*
|
|
760
|
+
* **New Options:**
|
|
761
|
+
* - β
`enableNeuralExtraction` - Extract entity names via AI
|
|
762
|
+
* - β
`enableConceptExtraction` - Extract entity types via AI
|
|
763
|
+
* - β
`preserveSource` - Save original file in VFS
|
|
764
|
+
*
|
|
765
|
+
* **If you get an error:**
|
|
766
|
+
* The error message includes migration instructions and examples.
|
|
767
|
+
* See the complete migration guide for all details.
|
|
768
|
+
*
|
|
769
|
+
* **Why these changes?**
|
|
770
|
+
* - Clearer option names (explicitly describe what they do)
|
|
771
|
+
* - Separation of concerns (neural, relationships, VFS are separate)
|
|
772
|
+
* - Better defaults (AI features enabled by default)
|
|
773
|
+
* - Reduced confusion (removed redundant options)
|
|
687
774
|
*/
|
|
688
775
|
import(source: Buffer | string | object, options?: {
|
|
689
776
|
format?: 'excel' | 'pdf' | 'csv' | 'json' | 'markdown';
|
package/dist/brainy.js
CHANGED
|
@@ -719,33 +719,75 @@ export class Brainy {
|
|
|
719
719
|
});
|
|
720
720
|
}
|
|
721
721
|
/**
|
|
722
|
-
* Get relationships
|
|
722
|
+
* Get relationships between entities
|
|
723
|
+
*
|
|
724
|
+
* Supports multiple query patterns:
|
|
725
|
+
* - No parameters: Returns all relationships (paginated, default limit: 100)
|
|
726
|
+
* - String ID: Returns relationships from that entity (shorthand for { from: id })
|
|
727
|
+
* - Parameters object: Fine-grained filtering and pagination
|
|
728
|
+
*
|
|
729
|
+
* @param paramsOrId - Optional string ID or parameters object
|
|
730
|
+
* @returns Promise resolving to array of relationships
|
|
731
|
+
*
|
|
732
|
+
* @example
|
|
733
|
+
* ```typescript
|
|
734
|
+
* // Get all relationships (first 100)
|
|
735
|
+
* const all = await brain.getRelations()
|
|
736
|
+
*
|
|
737
|
+
* // Get relationships from specific entity (shorthand syntax)
|
|
738
|
+
* const fromEntity = await brain.getRelations(entityId)
|
|
739
|
+
*
|
|
740
|
+
* // Get relationships with filters
|
|
741
|
+
* const filtered = await brain.getRelations({
|
|
742
|
+
* type: VerbType.FriendOf,
|
|
743
|
+
* limit: 50
|
|
744
|
+
* })
|
|
745
|
+
*
|
|
746
|
+
* // Pagination
|
|
747
|
+
* const page2 = await brain.getRelations({ offset: 100, limit: 100 })
|
|
748
|
+
* ```
|
|
749
|
+
*
|
|
750
|
+
* @since v4.1.3 - Fixed bug where calling without parameters returned empty array
|
|
751
|
+
* @since v4.1.3 - Added string ID shorthand syntax: getRelations(id)
|
|
723
752
|
*/
|
|
724
|
-
async getRelations(
|
|
753
|
+
async getRelations(paramsOrId) {
|
|
725
754
|
await this.ensureInitialized();
|
|
726
|
-
|
|
755
|
+
// Handle string ID shorthand: getRelations(id) -> getRelations({ from: id })
|
|
756
|
+
const params = typeof paramsOrId === 'string'
|
|
757
|
+
? { from: paramsOrId }
|
|
758
|
+
: (paramsOrId || {});
|
|
759
|
+
const limit = params.limit || 100;
|
|
760
|
+
const offset = params.offset || 0;
|
|
761
|
+
// Production safety: warn for large unfiltered queries
|
|
762
|
+
if (!params.from && !params.to && !params.type && limit > 10000) {
|
|
763
|
+
console.warn(`[Brainy] getRelations(): Fetching ${limit} relationships without filters. ` +
|
|
764
|
+
`Consider adding 'from', 'to', or 'type' filter for better performance.`);
|
|
765
|
+
}
|
|
766
|
+
// Build filter for storage query
|
|
767
|
+
const filter = {};
|
|
727
768
|
if (params.from) {
|
|
728
|
-
|
|
729
|
-
relations.push(...this.verbsToRelations(verbs));
|
|
769
|
+
filter.sourceId = params.from;
|
|
730
770
|
}
|
|
731
771
|
if (params.to) {
|
|
732
|
-
|
|
733
|
-
relations.push(...this.verbsToRelations(verbs));
|
|
772
|
+
filter.targetId = params.to;
|
|
734
773
|
}
|
|
735
|
-
// Filter by type
|
|
736
|
-
let filtered = relations;
|
|
737
774
|
if (params.type) {
|
|
738
|
-
|
|
739
|
-
filtered = relations.filter((r) => types.includes(r.type));
|
|
775
|
+
filter.verbType = Array.isArray(params.type) ? params.type : [params.type];
|
|
740
776
|
}
|
|
741
|
-
// Filter by service
|
|
742
777
|
if (params.service) {
|
|
743
|
-
|
|
744
|
-
}
|
|
745
|
-
//
|
|
746
|
-
const
|
|
747
|
-
|
|
748
|
-
|
|
778
|
+
filter.service = params.service;
|
|
779
|
+
}
|
|
780
|
+
// Fetch from storage with pagination at storage layer (efficient!)
|
|
781
|
+
const result = await this.storage.getVerbs({
|
|
782
|
+
pagination: {
|
|
783
|
+
limit,
|
|
784
|
+
offset,
|
|
785
|
+
cursor: params.cursor
|
|
786
|
+
},
|
|
787
|
+
filter: Object.keys(filter).length > 0 ? filter : undefined
|
|
788
|
+
});
|
|
789
|
+
// Convert to Relation format
|
|
790
|
+
return this.verbsToRelations(result.items);
|
|
749
791
|
}
|
|
750
792
|
// ============= SEARCH & DISCOVERY =============
|
|
751
793
|
/**
|
|
@@ -1551,33 +1593,91 @@ export class Brainy {
|
|
|
1551
1593
|
return options?.limit ? concepts.slice(0, options.limit) : concepts;
|
|
1552
1594
|
}
|
|
1553
1595
|
/**
|
|
1554
|
-
* Import files with
|
|
1596
|
+
* Import files with intelligent extraction and dual storage (VFS + Knowledge Graph)
|
|
1555
1597
|
*
|
|
1556
1598
|
* Unified import system that:
|
|
1557
1599
|
* - Auto-detects format (Excel, PDF, CSV, JSON, Markdown)
|
|
1558
|
-
* - Extracts entities
|
|
1600
|
+
* - Extracts entities with AI-powered name/type detection
|
|
1601
|
+
* - Infers semantic relationships from context
|
|
1559
1602
|
* - Stores in both VFS (organized files) and Knowledge Graph (connected entities)
|
|
1560
1603
|
* - Links VFS files to graph entities
|
|
1561
1604
|
*
|
|
1562
|
-
* @
|
|
1563
|
-
* // Import from file path
|
|
1564
|
-
* const result = await brain.import('/path/to/file.xlsx')
|
|
1605
|
+
* @since 4.0.0
|
|
1565
1606
|
*
|
|
1566
|
-
* @example
|
|
1567
|
-
*
|
|
1607
|
+
* @example Quick Start (All AI features enabled by default)
|
|
1608
|
+
* ```typescript
|
|
1609
|
+
* const result = await brain.import('./glossary.xlsx')
|
|
1610
|
+
* // Auto-detects format, extracts entities, infers relationships
|
|
1611
|
+
* ```
|
|
1612
|
+
*
|
|
1613
|
+
* @example Full-Featured Import (v4.x)
|
|
1614
|
+
* ```typescript
|
|
1615
|
+
* const result = await brain.import('./data.xlsx', {
|
|
1616
|
+
* // AI features
|
|
1617
|
+
* enableNeuralExtraction: true, // Extract entity names/metadata
|
|
1618
|
+
* enableRelationshipInference: true, // Detect semantic relationships
|
|
1619
|
+
* enableConceptExtraction: true, // Extract types/concepts
|
|
1620
|
+
*
|
|
1621
|
+
* // VFS features
|
|
1622
|
+
* vfsPath: '/imports/my-data', // Store in VFS directory
|
|
1623
|
+
* groupBy: 'type', // Organize by entity type
|
|
1624
|
+
* preserveSource: true, // Keep original file
|
|
1625
|
+
*
|
|
1626
|
+
* // Progress tracking
|
|
1627
|
+
* onProgress: (p) => console.log(p.message)
|
|
1628
|
+
* })
|
|
1629
|
+
* ```
|
|
1630
|
+
*
|
|
1631
|
+
* @example Performance Tuning (Large Files)
|
|
1632
|
+
* ```typescript
|
|
1633
|
+
* const result = await brain.import('./huge-file.csv', {
|
|
1634
|
+
* enableDeduplication: false, // Skip dedup for speed
|
|
1635
|
+
* confidenceThreshold: 0.8, // Higher threshold = fewer entities
|
|
1636
|
+
* onProgress: (p) => console.log(`${p.processed}/${p.total}`)
|
|
1637
|
+
* })
|
|
1638
|
+
* ```
|
|
1639
|
+
*
|
|
1640
|
+
* @example Import from Buffer or Object
|
|
1641
|
+
* ```typescript
|
|
1642
|
+
* // From buffer
|
|
1568
1643
|
* const result = await brain.import(buffer, { format: 'pdf' })
|
|
1569
1644
|
*
|
|
1570
|
-
*
|
|
1571
|
-
* // Import JSON object
|
|
1645
|
+
* // From object
|
|
1572
1646
|
* const result = await brain.import({ entities: [...] })
|
|
1647
|
+
* ```
|
|
1573
1648
|
*
|
|
1574
|
-
* @
|
|
1575
|
-
*
|
|
1576
|
-
*
|
|
1577
|
-
*
|
|
1578
|
-
*
|
|
1579
|
-
*
|
|
1580
|
-
*
|
|
1649
|
+
* @throws {Error} If invalid options are provided (v4.x breaking changes)
|
|
1650
|
+
*
|
|
1651
|
+
* @see {@link https://brainy.dev/docs/api/import API Documentation}
|
|
1652
|
+
* @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide}
|
|
1653
|
+
*
|
|
1654
|
+
* @remarks
|
|
1655
|
+
* **β οΈ Breaking Changes from v3.x:**
|
|
1656
|
+
*
|
|
1657
|
+
* The import API was redesigned in v4.0.0 for clarity and better feature control.
|
|
1658
|
+
* Old v3.x option names are **no longer recognized** and will throw errors.
|
|
1659
|
+
*
|
|
1660
|
+
* **Option Changes:**
|
|
1661
|
+
* - β `extractRelationships` β β
`enableRelationshipInference`
|
|
1662
|
+
* - β `createFileStructure` β β
`vfsPath: '/your/path'`
|
|
1663
|
+
* - β `autoDetect` β β
*(removed - always enabled)*
|
|
1664
|
+
* - β `excelSheets` β β
*(removed - all sheets processed)*
|
|
1665
|
+
* - β `pdfExtractTables` β β
*(removed - always enabled)*
|
|
1666
|
+
*
|
|
1667
|
+
* **New Options:**
|
|
1668
|
+
* - β
`enableNeuralExtraction` - Extract entity names via AI
|
|
1669
|
+
* - β
`enableConceptExtraction` - Extract entity types via AI
|
|
1670
|
+
* - β
`preserveSource` - Save original file in VFS
|
|
1671
|
+
*
|
|
1672
|
+
* **If you get an error:**
|
|
1673
|
+
* The error message includes migration instructions and examples.
|
|
1674
|
+
* See the complete migration guide for all details.
|
|
1675
|
+
*
|
|
1676
|
+
* **Why these changes?**
|
|
1677
|
+
* - Clearer option names (explicitly describe what they do)
|
|
1678
|
+
* - Separation of concerns (neural, relationships, VFS are separate)
|
|
1679
|
+
* - Better defaults (AI features enabled by default)
|
|
1680
|
+
* - Reduced confusion (removed redundant options)
|
|
1581
1681
|
*/
|
|
1582
1682
|
async import(source, options) {
|
|
1583
1683
|
// Lazy load ImportCoordinator
|
|
@@ -21,7 +21,10 @@ export interface ImportSource {
|
|
|
21
21
|
/** Optional filename hint */
|
|
22
22
|
filename?: string;
|
|
23
23
|
}
|
|
24
|
-
|
|
24
|
+
/**
|
|
25
|
+
* Valid import options for v4.x
|
|
26
|
+
*/
|
|
27
|
+
export interface ValidImportOptions {
|
|
25
28
|
/** Force specific format (skip auto-detection) */
|
|
26
29
|
format?: SupportedFormat;
|
|
27
30
|
/** VFS root path for imported files */
|
|
@@ -55,6 +58,45 @@ export interface ImportOptions {
|
|
|
55
58
|
/** Progress callback */
|
|
56
59
|
onProgress?: (progress: ImportProgress) => void;
|
|
57
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* Deprecated import options from v3.x
|
|
63
|
+
* Using these will cause TypeScript compile errors
|
|
64
|
+
*
|
|
65
|
+
* @deprecated These options are no longer supported in v4.x
|
|
66
|
+
* @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide}
|
|
67
|
+
*/
|
|
68
|
+
export interface DeprecatedImportOptions {
|
|
69
|
+
/**
|
|
70
|
+
* @deprecated Use `enableRelationshipInference` instead
|
|
71
|
+
* @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide}
|
|
72
|
+
*/
|
|
73
|
+
extractRelationships?: never;
|
|
74
|
+
/**
|
|
75
|
+
* @deprecated Removed in v4.x - auto-detection is now always enabled
|
|
76
|
+
* @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide}
|
|
77
|
+
*/
|
|
78
|
+
autoDetect?: never;
|
|
79
|
+
/**
|
|
80
|
+
* @deprecated Use `vfsPath` to specify the directory path instead
|
|
81
|
+
* @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide}
|
|
82
|
+
*/
|
|
83
|
+
createFileStructure?: never;
|
|
84
|
+
/**
|
|
85
|
+
* @deprecated Removed in v4.x - all sheets are now processed automatically
|
|
86
|
+
* @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide}
|
|
87
|
+
*/
|
|
88
|
+
excelSheets?: never;
|
|
89
|
+
/**
|
|
90
|
+
* @deprecated Removed in v4.x - table extraction is now automatic for PDF imports
|
|
91
|
+
* @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide}
|
|
92
|
+
*/
|
|
93
|
+
pdfExtractTables?: never;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Complete import options interface
|
|
97
|
+
* Combines valid v4.x options with deprecated v3.x options (which cause TypeScript errors)
|
|
98
|
+
*/
|
|
99
|
+
export type ImportOptions = ValidImportOptions & DeprecatedImportOptions;
|
|
58
100
|
export interface ImportProgress {
|
|
59
101
|
stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'relationships' | 'complete';
|
|
60
102
|
/** Phase of import - extraction or relationship building (v3.49.0) */
|
|
@@ -165,4 +207,14 @@ export declare class ImportCoordinator {
|
|
|
165
207
|
* Normalize extraction result to unified format (Excel-like structure)
|
|
166
208
|
*/
|
|
167
209
|
private normalizeExtractionResult;
|
|
210
|
+
/**
|
|
211
|
+
* Validate options and reject deprecated v3.x options (v4.0.0+)
|
|
212
|
+
* Throws clear errors with migration guidance
|
|
213
|
+
*/
|
|
214
|
+
private validateOptions;
|
|
215
|
+
/**
|
|
216
|
+
* Build detailed error message for invalid options
|
|
217
|
+
* Respects LOG_LEVEL for verbosity (detailed in dev, concise in prod)
|
|
218
|
+
*/
|
|
219
|
+
private buildValidationErrorMessage;
|
|
168
220
|
}
|
|
@@ -62,6 +62,8 @@ export class ImportCoordinator {
|
|
|
62
62
|
async import(source, options = {}) {
|
|
63
63
|
const startTime = Date.now();
|
|
64
64
|
const importId = uuidv4();
|
|
65
|
+
// Validate options (v4.0.0+: Reject deprecated v3.x options)
|
|
66
|
+
this.validateOptions(options);
|
|
65
67
|
// Normalize source
|
|
66
68
|
const normalizedSource = this.normalizeSource(source, options.format);
|
|
67
69
|
// Report detection stage
|
|
@@ -558,5 +560,101 @@ export class ImportCoordinator {
|
|
|
558
560
|
// Fallback: return as-is
|
|
559
561
|
return result;
|
|
560
562
|
}
|
|
563
|
+
/**
|
|
564
|
+
* Validate options and reject deprecated v3.x options (v4.0.0+)
|
|
565
|
+
* Throws clear errors with migration guidance
|
|
566
|
+
*/
|
|
567
|
+
validateOptions(options) {
|
|
568
|
+
const invalidOptions = [];
|
|
569
|
+
// Check for v3.x deprecated options
|
|
570
|
+
if ('extractRelationships' in options) {
|
|
571
|
+
invalidOptions.push({
|
|
572
|
+
old: 'extractRelationships',
|
|
573
|
+
new: 'enableRelationshipInference',
|
|
574
|
+
message: 'Option renamed for clarity in v4.x - explicitly indicates AI-powered relationship inference'
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
if ('autoDetect' in options) {
|
|
578
|
+
invalidOptions.push({
|
|
579
|
+
old: 'autoDetect',
|
|
580
|
+
new: '(removed)',
|
|
581
|
+
message: 'Auto-detection is now always enabled - no need to specify this option'
|
|
582
|
+
});
|
|
583
|
+
}
|
|
584
|
+
if ('createFileStructure' in options) {
|
|
585
|
+
invalidOptions.push({
|
|
586
|
+
old: 'createFileStructure',
|
|
587
|
+
new: 'vfsPath',
|
|
588
|
+
message: 'Use vfsPath to explicitly specify the virtual filesystem directory path'
|
|
589
|
+
});
|
|
590
|
+
}
|
|
591
|
+
if ('excelSheets' in options) {
|
|
592
|
+
invalidOptions.push({
|
|
593
|
+
old: 'excelSheets',
|
|
594
|
+
new: '(removed)',
|
|
595
|
+
message: 'All sheets are now processed automatically - no configuration needed'
|
|
596
|
+
});
|
|
597
|
+
}
|
|
598
|
+
if ('pdfExtractTables' in options) {
|
|
599
|
+
invalidOptions.push({
|
|
600
|
+
old: 'pdfExtractTables',
|
|
601
|
+
new: '(removed)',
|
|
602
|
+
message: 'Table extraction is now automatic for PDF imports'
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
// If invalid options found, throw error with detailed message
|
|
606
|
+
if (invalidOptions.length > 0) {
|
|
607
|
+
const errorMessage = this.buildValidationErrorMessage(invalidOptions);
|
|
608
|
+
throw new Error(errorMessage);
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* Build detailed error message for invalid options
|
|
613
|
+
* Respects LOG_LEVEL for verbosity (detailed in dev, concise in prod)
|
|
614
|
+
*/
|
|
615
|
+
buildValidationErrorMessage(invalidOptions) {
|
|
616
|
+
// Check environment for verbosity level
|
|
617
|
+
const verbose = process.env.LOG_LEVEL === 'debug' ||
|
|
618
|
+
process.env.LOG_LEVEL === 'verbose' ||
|
|
619
|
+
process.env.NODE_ENV === 'development' ||
|
|
620
|
+
process.env.NODE_ENV === 'dev';
|
|
621
|
+
if (verbose) {
|
|
622
|
+
// DETAILED mode (development)
|
|
623
|
+
const optionDetails = invalidOptions
|
|
624
|
+
.map((opt) => `
|
|
625
|
+
β ${opt.old}
|
|
626
|
+
β Use: ${opt.new}
|
|
627
|
+
β Why: ${opt.message}`)
|
|
628
|
+
.join('\n');
|
|
629
|
+
return `
|
|
630
|
+
β Invalid import options detected (Brainy v4.x breaking changes)
|
|
631
|
+
|
|
632
|
+
The following v3.x options are no longer supported:
|
|
633
|
+
${optionDetails}
|
|
634
|
+
|
|
635
|
+
π Migration Guide: https://brainy.dev/docs/guides/migrating-to-v4
|
|
636
|
+
π‘ Quick Fix Examples:
|
|
637
|
+
|
|
638
|
+
Before (v3.x):
|
|
639
|
+
await brain.import(file, {
|
|
640
|
+
extractRelationships: true,
|
|
641
|
+
createFileStructure: true
|
|
642
|
+
})
|
|
643
|
+
|
|
644
|
+
After (v4.x):
|
|
645
|
+
await brain.import(file, {
|
|
646
|
+
enableRelationshipInference: true,
|
|
647
|
+
vfsPath: '/imports/my-data'
|
|
648
|
+
})
|
|
649
|
+
|
|
650
|
+
π Full API docs: https://brainy.dev/docs/api/import
|
|
651
|
+
`.trim();
|
|
652
|
+
}
|
|
653
|
+
else {
|
|
654
|
+
// CONCISE mode (production)
|
|
655
|
+
const optionsList = invalidOptions.map((o) => `'${o.old}'`).join(', ');
|
|
656
|
+
return `Invalid import options: ${optionsList}. See https://brainy.dev/docs/guides/migrating-to-v4`;
|
|
657
|
+
}
|
|
658
|
+
}
|
|
561
659
|
}
|
|
562
660
|
//# sourceMappingURL=ImportCoordinator.js.map
|
|
@@ -1310,14 +1310,14 @@ export class ImprovedNeuralAPI {
|
|
|
1310
1310
|
for (const sourceId of itemIds) {
|
|
1311
1311
|
const sourceVerbs = await this.brain.getRelations(sourceId);
|
|
1312
1312
|
for (const verb of sourceVerbs) {
|
|
1313
|
-
const targetId = verb.
|
|
1313
|
+
const targetId = verb.to;
|
|
1314
1314
|
if (nodes.has(targetId) && sourceId !== targetId) {
|
|
1315
1315
|
// Initialize edge map if needed
|
|
1316
1316
|
if (!edges.has(sourceId)) {
|
|
1317
1317
|
edges.set(sourceId, new Map());
|
|
1318
1318
|
}
|
|
1319
1319
|
// Calculate edge weight from verb type and metadata
|
|
1320
|
-
const verbType = verb.
|
|
1320
|
+
const verbType = verb.type;
|
|
1321
1321
|
const baseWeight = relationshipWeights[verbType] || 0.5;
|
|
1322
1322
|
const confidenceWeight = verb.confidence || 1.0;
|
|
1323
1323
|
const weight = baseWeight * confidenceWeight;
|
|
@@ -2743,7 +2743,7 @@ export class ImprovedNeuralAPI {
|
|
|
2743
2743
|
const sampleSize = Math.min(50, itemIds.length);
|
|
2744
2744
|
for (let i = 0; i < sampleSize; i++) {
|
|
2745
2745
|
try {
|
|
2746
|
-
const verbs = await this.brain.
|
|
2746
|
+
const verbs = await this.brain.getRelations({ from: itemIds[i] });
|
|
2747
2747
|
connectionCount += verbs.length;
|
|
2748
2748
|
}
|
|
2749
2749
|
catch (error) {
|
|
@@ -2797,9 +2797,9 @@ export class ImprovedNeuralAPI {
|
|
|
2797
2797
|
if (fromType !== toType) {
|
|
2798
2798
|
for (const fromItem of fromItems.slice(0, 10)) { // Sample to avoid N^2
|
|
2799
2799
|
try {
|
|
2800
|
-
const verbs = await this.brain.
|
|
2800
|
+
const verbs = await this.brain.getRelations({ from: fromItem.id });
|
|
2801
2801
|
for (const verb of verbs) {
|
|
2802
|
-
const toItem = toItems.find(item => item.id === verb.
|
|
2802
|
+
const toItem = toItems.find(item => item.id === verb.to);
|
|
2803
2803
|
if (toItem) {
|
|
2804
2804
|
connections.push({
|
|
2805
2805
|
from: fromItem.id,
|
|
@@ -600,13 +600,15 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
600
600
|
// Check if the adapter has a paginated method for getting verbs
|
|
601
601
|
if (typeof this.getVerbsWithPagination === 'function') {
|
|
602
602
|
// Use the adapter's paginated method
|
|
603
|
+
// Convert offset to cursor if no cursor provided (adapters use cursor for offset)
|
|
604
|
+
const effectiveCursor = cursor || (offset > 0 ? offset.toString() : undefined);
|
|
603
605
|
const result = await this.getVerbsWithPagination({
|
|
604
606
|
limit,
|
|
605
|
-
cursor,
|
|
607
|
+
cursor: effectiveCursor,
|
|
606
608
|
filter: options?.filter
|
|
607
609
|
});
|
|
608
|
-
//
|
|
609
|
-
const items = result.items
|
|
610
|
+
// Items are already offset by the adapter via cursor, no need to slice
|
|
611
|
+
const items = result.items;
|
|
610
612
|
// CRITICAL SAFETY CHECK: Prevent infinite loops
|
|
611
613
|
// If we have no items but hasMore is true, force hasMore to false
|
|
612
614
|
// This prevents pagination bugs from causing infinite loops
|
|
@@ -172,14 +172,83 @@ export interface SimilarParams<T = any> {
|
|
|
172
172
|
}
|
|
173
173
|
/**
|
|
174
174
|
* Parameters for getting relationships
|
|
175
|
+
*
|
|
176
|
+
* All parameters are optional. When called without parameters, returns all relationships
|
|
177
|
+
* with pagination (default limit: 100).
|
|
178
|
+
*
|
|
179
|
+
* @example
|
|
180
|
+
* ```typescript
|
|
181
|
+
* // Get all relationships (default limit: 100)
|
|
182
|
+
* const all = await brain.getRelations()
|
|
183
|
+
*
|
|
184
|
+
* // Get relationships from a specific entity (string shorthand)
|
|
185
|
+
* const fromEntity = await brain.getRelations(entityId)
|
|
186
|
+
*
|
|
187
|
+
* // Equivalent to:
|
|
188
|
+
* const fromEntity2 = await brain.getRelations({ from: entityId })
|
|
189
|
+
*
|
|
190
|
+
* // Get relationships to a specific entity
|
|
191
|
+
* const toEntity = await brain.getRelations({ to: entityId })
|
|
192
|
+
*
|
|
193
|
+
* // Filter by relationship type
|
|
194
|
+
* const friends = await brain.getRelations({ type: VerbType.FriendOf })
|
|
195
|
+
*
|
|
196
|
+
* // Pagination
|
|
197
|
+
* const page2 = await brain.getRelations({ offset: 100, limit: 50 })
|
|
198
|
+
*
|
|
199
|
+
* // Combined filters
|
|
200
|
+
* const filtered = await brain.getRelations({
|
|
201
|
+
* from: entityId,
|
|
202
|
+
* type: VerbType.WorksWith,
|
|
203
|
+
* limit: 20
|
|
204
|
+
* })
|
|
205
|
+
* ```
|
|
206
|
+
*
|
|
207
|
+
* @since v4.1.3 - Fixed bug where calling without parameters returned empty array
|
|
208
|
+
* @since v4.1.3 - Added string ID shorthand syntax
|
|
175
209
|
*/
|
|
176
210
|
export interface GetRelationsParams {
|
|
211
|
+
/**
|
|
212
|
+
* Filter by source entity ID
|
|
213
|
+
*
|
|
214
|
+
* Returns all relationships originating from this entity.
|
|
215
|
+
*/
|
|
177
216
|
from?: string;
|
|
217
|
+
/**
|
|
218
|
+
* Filter by target entity ID
|
|
219
|
+
*
|
|
220
|
+
* Returns all relationships pointing to this entity.
|
|
221
|
+
*/
|
|
178
222
|
to?: string;
|
|
223
|
+
/**
|
|
224
|
+
* Filter by relationship type(s)
|
|
225
|
+
*
|
|
226
|
+
* Can be a single VerbType or array of VerbTypes.
|
|
227
|
+
*/
|
|
179
228
|
type?: VerbType | VerbType[];
|
|
229
|
+
/**
|
|
230
|
+
* Maximum number of results to return
|
|
231
|
+
*
|
|
232
|
+
* @default 100
|
|
233
|
+
*/
|
|
180
234
|
limit?: number;
|
|
235
|
+
/**
|
|
236
|
+
* Number of results to skip (offset-based pagination)
|
|
237
|
+
*
|
|
238
|
+
* @default 0
|
|
239
|
+
*/
|
|
181
240
|
offset?: number;
|
|
241
|
+
/**
|
|
242
|
+
* Cursor for cursor-based pagination
|
|
243
|
+
*
|
|
244
|
+
* More efficient than offset for large result sets.
|
|
245
|
+
*/
|
|
182
246
|
cursor?: string;
|
|
247
|
+
/**
|
|
248
|
+
* Filter by service (multi-tenancy)
|
|
249
|
+
*
|
|
250
|
+
* Only return relationships belonging to this service.
|
|
251
|
+
*/
|
|
183
252
|
service?: string;
|
|
184
253
|
}
|
|
185
254
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soulcraft/brainy",
|
|
3
|
-
"version": "4.1.
|
|
3
|
+
"version": "4.1.4",
|
|
4
4
|
"description": "Universal Knowledge Protocolβ’ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns Γ 40 verbs for infinite expressiveness.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.js",
|