@soulcraft/brainy 3.50.1 ā 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +242 -0
- package/README.md +358 -658
- package/dist/api/ConfigAPI.js +56 -19
- package/dist/api/DataAPI.js +24 -18
- package/dist/augmentations/storageAugmentations.d.ts +24 -0
- package/dist/augmentations/storageAugmentations.js +22 -0
- package/dist/brainy.js +32 -9
- package/dist/cli/commands/core.d.ts +20 -10
- package/dist/cli/commands/core.js +384 -82
- package/dist/cli/commands/import.d.ts +41 -0
- package/dist/cli/commands/import.js +456 -0
- package/dist/cli/commands/insights.d.ts +34 -0
- package/dist/cli/commands/insights.js +300 -0
- package/dist/cli/commands/neural.d.ts +6 -12
- package/dist/cli/commands/neural.js +113 -10
- package/dist/cli/commands/nlp.d.ts +28 -0
- package/dist/cli/commands/nlp.js +246 -0
- package/dist/cli/commands/storage.d.ts +64 -0
- package/dist/cli/commands/storage.js +730 -0
- package/dist/cli/index.js +210 -24
- package/dist/coreTypes.d.ts +206 -34
- package/dist/distributed/configManager.js +8 -6
- package/dist/distributed/shardMigration.js +2 -0
- package/dist/distributed/storageDiscovery.js +6 -4
- package/dist/embeddings/EmbeddingManager.d.ts +2 -2
- package/dist/embeddings/EmbeddingManager.js +5 -1
- package/dist/graph/lsm/LSMTree.js +32 -20
- package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
- package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
- package/dist/storage/adapters/azureBlobStorage.js +1809 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
- package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
- package/dist/storage/adapters/fileSystemStorage.js +204 -127
- package/dist/storage/adapters/gcsStorage.d.ts +119 -9
- package/dist/storage/adapters/gcsStorage.js +317 -62
- package/dist/storage/adapters/memoryStorage.d.ts +30 -18
- package/dist/storage/adapters/memoryStorage.js +99 -94
- package/dist/storage/adapters/opfsStorage.d.ts +48 -10
- package/dist/storage/adapters/opfsStorage.js +201 -80
- package/dist/storage/adapters/r2Storage.d.ts +12 -5
- package/dist/storage/adapters/r2Storage.js +63 -15
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
- package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
- package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
- package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
- package/dist/storage/baseStorage.d.ts +41 -38
- package/dist/storage/baseStorage.js +110 -134
- package/dist/storage/storageFactory.d.ts +29 -2
- package/dist/storage/storageFactory.js +30 -1
- package/dist/utils/entityIdMapper.js +5 -2
- package/dist/utils/fieldTypeInference.js +8 -1
- package/dist/utils/metadataFilter.d.ts +3 -2
- package/dist/utils/metadataFilter.js +1 -0
- package/dist/utils/metadataIndex.d.ts +2 -1
- package/dist/utils/metadataIndex.js +9 -1
- package/dist/utils/metadataIndexChunking.js +9 -4
- package/dist/utils/periodicCleanup.js +1 -0
- package/package.json +3 -1
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NLP Commands - Natural Language Processing
|
|
3
|
+
*
|
|
4
|
+
* Extract entities, concepts, and insights from text using Brainy's neural NLP
|
|
5
|
+
*/
|
|
6
|
+
import chalk from 'chalk';
|
|
7
|
+
import ora from 'ora';
|
|
8
|
+
import inquirer from 'inquirer';
|
|
9
|
+
import Table from 'cli-table3';
|
|
10
|
+
import { Brainy } from '../../brainy.js';
|
|
11
|
+
let brainyInstance = null;
|
|
12
|
+
const getBrainy = () => {
|
|
13
|
+
if (!brainyInstance) {
|
|
14
|
+
brainyInstance = new Brainy();
|
|
15
|
+
}
|
|
16
|
+
return brainyInstance;
|
|
17
|
+
};
|
|
18
|
+
const formatOutput = (data, options) => {
|
|
19
|
+
if (options.json) {
|
|
20
|
+
console.log(options.pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data));
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
export const nlpCommands = {
|
|
24
|
+
/**
|
|
25
|
+
* Extract entities from text
|
|
26
|
+
*/
|
|
27
|
+
async extract(text, options) {
|
|
28
|
+
let spinner = null;
|
|
29
|
+
try {
|
|
30
|
+
// Interactive mode if no text provided
|
|
31
|
+
if (!text) {
|
|
32
|
+
const answers = await inquirer.prompt([
|
|
33
|
+
{
|
|
34
|
+
type: 'editor',
|
|
35
|
+
name: 'text',
|
|
36
|
+
message: 'Enter or paste text to analyze (will open editor):',
|
|
37
|
+
validate: (input) => input.trim().length > 0 || 'Text cannot be empty'
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
type: 'confirm',
|
|
41
|
+
name: 'saveEntities',
|
|
42
|
+
message: 'Save extracted entities to database?',
|
|
43
|
+
default: false
|
|
44
|
+
}
|
|
45
|
+
]);
|
|
46
|
+
text = answers.text;
|
|
47
|
+
options = { ...options, ...(answers.saveEntities && { save: true }) };
|
|
48
|
+
}
|
|
49
|
+
spinner = ora('Extracting entities with neural NLP...').start();
|
|
50
|
+
const brain = getBrainy();
|
|
51
|
+
// Extract entities using Brainy's neural entity extractor
|
|
52
|
+
const entities = await brain.extract(text);
|
|
53
|
+
spinner.succeed(`Extracted ${entities.length} entities`);
|
|
54
|
+
if (!options.json) {
|
|
55
|
+
if (entities.length === 0) {
|
|
56
|
+
console.log(chalk.yellow('\nNo entities found'));
|
|
57
|
+
console.log(chalk.dim('Try providing more specific or detailed text'));
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
console.log(chalk.cyan(`\nš§ Extracted ${entities.length} Entities:\n`));
|
|
61
|
+
const table = new Table({
|
|
62
|
+
head: [chalk.cyan('Type'), chalk.cyan('Entity'), chalk.cyan('Confidence')],
|
|
63
|
+
colWidths: [15, 40, 15]
|
|
64
|
+
});
|
|
65
|
+
entities.forEach((entity) => {
|
|
66
|
+
table.push([
|
|
67
|
+
entity.type || 'Unknown',
|
|
68
|
+
entity.content || entity.text || entity.value,
|
|
69
|
+
`${((entity.confidence || 0) * 100).toFixed(1)}%`
|
|
70
|
+
]);
|
|
71
|
+
});
|
|
72
|
+
console.log(table.toString());
|
|
73
|
+
// Show summary by type
|
|
74
|
+
const byType = entities.reduce((acc, e) => {
|
|
75
|
+
const type = e.type || 'Unknown';
|
|
76
|
+
acc[type] = (acc[type] || 0) + 1;
|
|
77
|
+
return acc;
|
|
78
|
+
}, {});
|
|
79
|
+
console.log(chalk.cyan('\nš Summary by Type:'));
|
|
80
|
+
Object.entries(byType).forEach(([type, count]) => {
|
|
81
|
+
console.log(` ${type}: ${chalk.yellow(count)}`);
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
formatOutput(entities, options);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
if (spinner)
|
|
91
|
+
spinner.fail('Entity extraction failed');
|
|
92
|
+
console.error(chalk.red('Extraction failed:', error.message));
|
|
93
|
+
if (options.verbose) {
|
|
94
|
+
console.error(chalk.dim(error.stack));
|
|
95
|
+
}
|
|
96
|
+
process.exit(1);
|
|
97
|
+
}
|
|
98
|
+
},
|
|
99
|
+
/**
|
|
100
|
+
* Extract concepts from text
|
|
101
|
+
*/
|
|
102
|
+
async extractConcepts(text, options) {
|
|
103
|
+
let spinner = null;
|
|
104
|
+
try {
|
|
105
|
+
// Interactive mode if no text provided
|
|
106
|
+
if (!text) {
|
|
107
|
+
const answers = await inquirer.prompt([
|
|
108
|
+
{
|
|
109
|
+
type: 'editor',
|
|
110
|
+
name: 'text',
|
|
111
|
+
message: 'Enter or paste text to analyze:',
|
|
112
|
+
validate: (input) => input.trim().length > 0 || 'Text cannot be empty'
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
type: 'number',
|
|
116
|
+
name: 'threshold',
|
|
117
|
+
message: 'Minimum confidence threshold (0-1):',
|
|
118
|
+
default: 0.5,
|
|
119
|
+
validate: (input) => (input >= 0 && input <= 1) || 'Must be between 0 and 1'
|
|
120
|
+
}
|
|
121
|
+
]);
|
|
122
|
+
text = answers.text;
|
|
123
|
+
if (!options.threshold) {
|
|
124
|
+
options.threshold = answers.threshold.toString();
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
spinner = ora('Extracting concepts with neural analysis...').start();
|
|
128
|
+
const brain = getBrainy();
|
|
129
|
+
const confidence = options.threshold ? parseFloat(options.threshold) : 0.5;
|
|
130
|
+
const concepts = await brain.extractConcepts(text, { confidence });
|
|
131
|
+
spinner.succeed(`Extracted ${concepts.length} concepts`);
|
|
132
|
+
if (!options.json) {
|
|
133
|
+
if (concepts.length === 0) {
|
|
134
|
+
console.log(chalk.yellow('\nNo concepts found above threshold'));
|
|
135
|
+
console.log(chalk.dim(`Try lowering the threshold (currently ${confidence})`));
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
console.log(chalk.cyan(`\nš” Extracted ${concepts.length} Concepts:\n`));
|
|
139
|
+
// concepts is string[] - display as simple list
|
|
140
|
+
concepts.forEach((concept, index) => {
|
|
141
|
+
console.log(` ${chalk.yellow(`${index + 1}.`)} ${concept}`);
|
|
142
|
+
});
|
|
143
|
+
console.log(chalk.dim(`\nš” Confidence threshold: ${confidence} (${(confidence * 100).toFixed(0)}% minimum)`));
|
|
144
|
+
console.log(chalk.dim(` Higher threshold = fewer but more relevant concepts`));
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
formatOutput(concepts, options);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
if (spinner)
|
|
153
|
+
spinner.fail('Concept extraction failed');
|
|
154
|
+
console.error(chalk.red('Extraction failed:', error.message));
|
|
155
|
+
if (options.verbose) {
|
|
156
|
+
console.error(chalk.dim(error.stack));
|
|
157
|
+
}
|
|
158
|
+
process.exit(1);
|
|
159
|
+
}
|
|
160
|
+
},
|
|
161
|
+
/**
|
|
162
|
+
* Analyze text with full NLP pipeline
|
|
163
|
+
*/
|
|
164
|
+
async analyze(text, options) {
|
|
165
|
+
let spinner = null;
|
|
166
|
+
try {
|
|
167
|
+
// Interactive mode if no text provided
|
|
168
|
+
if (!text) {
|
|
169
|
+
const answer = await inquirer.prompt([{
|
|
170
|
+
type: 'editor',
|
|
171
|
+
name: 'text',
|
|
172
|
+
message: 'Enter or paste text to analyze:',
|
|
173
|
+
validate: (input) => input.trim().length > 0 || 'Text cannot be empty'
|
|
174
|
+
}]);
|
|
175
|
+
text = answer.text;
|
|
176
|
+
}
|
|
177
|
+
spinner = ora('Analyzing text with neural NLP...').start();
|
|
178
|
+
const brain = getBrainy();
|
|
179
|
+
// Run both entity extraction and concept extraction
|
|
180
|
+
const [entities, concepts] = await Promise.all([
|
|
181
|
+
brain.extract(text),
|
|
182
|
+
brain.extractConcepts(text, { confidence: 0.5 })
|
|
183
|
+
]);
|
|
184
|
+
spinner.succeed('Analysis complete');
|
|
185
|
+
if (!options.json) {
|
|
186
|
+
console.log(chalk.cyan('\nš§ NLP Analysis Results:\n'));
|
|
187
|
+
// Text summary
|
|
188
|
+
const wordCount = text.split(/\s+/).length;
|
|
189
|
+
const charCount = text.length;
|
|
190
|
+
console.log(chalk.bold('š Text Summary:'));
|
|
191
|
+
console.log(` Characters: ${chalk.yellow(charCount)}`);
|
|
192
|
+
console.log(` Words: ${chalk.yellow(wordCount)}`);
|
|
193
|
+
console.log(` Avg word length: ${chalk.yellow((charCount / wordCount).toFixed(1))}`);
|
|
194
|
+
// Entities
|
|
195
|
+
console.log(chalk.bold('\nš Entities Detected:'), chalk.yellow(entities.length));
|
|
196
|
+
if (entities.length > 0) {
|
|
197
|
+
const table = new Table({
|
|
198
|
+
head: [chalk.cyan('Entity'), chalk.cyan('Type'), chalk.cyan('Confidence')],
|
|
199
|
+
colWidths: [40, 20, 15]
|
|
200
|
+
});
|
|
201
|
+
entities.slice(0, 10).forEach((e) => {
|
|
202
|
+
table.push([
|
|
203
|
+
e.content || e.text || 'Unknown',
|
|
204
|
+
e.type || 'Unknown',
|
|
205
|
+
`${((e.confidence || 0) * 100).toFixed(1)}%`
|
|
206
|
+
]);
|
|
207
|
+
});
|
|
208
|
+
console.log(table.toString());
|
|
209
|
+
if (entities.length > 10) {
|
|
210
|
+
console.log(chalk.dim(`\n... and ${entities.length - 10} more entities`));
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
// Concepts
|
|
214
|
+
if (concepts.length > 0) {
|
|
215
|
+
console.log(chalk.bold('\nš” Key Concepts:'));
|
|
216
|
+
concepts.slice(0, 10).forEach((concept, index) => {
|
|
217
|
+
console.log(` ${chalk.yellow(`${index + 1}.`)} ${concept}`);
|
|
218
|
+
});
|
|
219
|
+
if (concepts.length > 10) {
|
|
220
|
+
console.log(chalk.dim(` ... and ${concepts.length - 10} more`));
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
else {
|
|
225
|
+
formatOutput({
|
|
226
|
+
text: {
|
|
227
|
+
length: text.length,
|
|
228
|
+
wordCount: text.split(/\s+/).length
|
|
229
|
+
},
|
|
230
|
+
entities,
|
|
231
|
+
concepts
|
|
232
|
+
}, options);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
catch (error) {
|
|
236
|
+
if (spinner)
|
|
237
|
+
spinner.fail('Analysis failed');
|
|
238
|
+
console.error(chalk.red('Analysis failed:', error.message));
|
|
239
|
+
if (options.verbose) {
|
|
240
|
+
console.error(chalk.dim(error.stack));
|
|
241
|
+
}
|
|
242
|
+
process.exit(1);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
};
|
|
246
|
+
//# sourceMappingURL=nlp.js.map
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* š¾ Storage Management Commands - v4.0.0
|
|
3
|
+
*
|
|
4
|
+
* Modern interactive CLI for storage lifecycle, cost optimization, and management
|
|
5
|
+
*/
|
|
6
|
+
interface StorageOptions {
|
|
7
|
+
verbose?: boolean;
|
|
8
|
+
json?: boolean;
|
|
9
|
+
pretty?: boolean;
|
|
10
|
+
}
|
|
11
|
+
export declare const storageCommands: {
|
|
12
|
+
/**
|
|
13
|
+
* Show storage status and health
|
|
14
|
+
*/
|
|
15
|
+
status(options: StorageOptions & {
|
|
16
|
+
detailed?: boolean;
|
|
17
|
+
quota?: boolean;
|
|
18
|
+
}): Promise<void>;
|
|
19
|
+
/**
|
|
20
|
+
* Lifecycle policy management
|
|
21
|
+
*/
|
|
22
|
+
lifecycle: {
|
|
23
|
+
/**
|
|
24
|
+
* Set lifecycle policy (interactive or from file)
|
|
25
|
+
*/
|
|
26
|
+
set(configFile?: string, options?: StorageOptions & {
|
|
27
|
+
validate?: boolean;
|
|
28
|
+
}): Promise<void>;
|
|
29
|
+
/**
|
|
30
|
+
* Get current lifecycle policy
|
|
31
|
+
*/
|
|
32
|
+
get(options?: StorageOptions & {
|
|
33
|
+
format?: "json" | "yaml";
|
|
34
|
+
}): Promise<void>;
|
|
35
|
+
/**
|
|
36
|
+
* Remove lifecycle policy
|
|
37
|
+
*/
|
|
38
|
+
remove(options: StorageOptions): Promise<void>;
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* Compression management (FileSystem storage)
|
|
42
|
+
*/
|
|
43
|
+
compression: {
|
|
44
|
+
enable(options: StorageOptions): Promise<void>;
|
|
45
|
+
disable(options: StorageOptions): Promise<void>;
|
|
46
|
+
status(options: StorageOptions): Promise<void>;
|
|
47
|
+
};
|
|
48
|
+
/**
|
|
49
|
+
* Batch delete with retry logic
|
|
50
|
+
*/
|
|
51
|
+
batchDelete(file: string, options?: StorageOptions & {
|
|
52
|
+
maxRetries?: string;
|
|
53
|
+
continueOnError?: boolean;
|
|
54
|
+
}): Promise<void>;
|
|
55
|
+
/**
|
|
56
|
+
* Cost estimation tool
|
|
57
|
+
*/
|
|
58
|
+
costEstimate(options?: StorageOptions & {
|
|
59
|
+
provider?: "aws" | "gcs" | "azure" | "r2";
|
|
60
|
+
size?: string;
|
|
61
|
+
operations?: string;
|
|
62
|
+
}): Promise<void>;
|
|
63
|
+
};
|
|
64
|
+
export {};
|