@soulcraft/brainy 3.50.2 ā 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +201 -0
- package/README.md +358 -658
- package/dist/api/ConfigAPI.js +56 -19
- package/dist/api/DataAPI.js +24 -18
- package/dist/augmentations/storageAugmentations.d.ts +24 -0
- package/dist/augmentations/storageAugmentations.js +22 -0
- package/dist/brainy.js +32 -9
- package/dist/cli/commands/core.d.ts +20 -10
- package/dist/cli/commands/core.js +384 -82
- package/dist/cli/commands/import.d.ts +41 -0
- package/dist/cli/commands/import.js +456 -0
- package/dist/cli/commands/insights.d.ts +34 -0
- package/dist/cli/commands/insights.js +300 -0
- package/dist/cli/commands/neural.d.ts +6 -12
- package/dist/cli/commands/neural.js +113 -10
- package/dist/cli/commands/nlp.d.ts +28 -0
- package/dist/cli/commands/nlp.js +246 -0
- package/dist/cli/commands/storage.d.ts +64 -0
- package/dist/cli/commands/storage.js +730 -0
- package/dist/cli/index.js +210 -24
- package/dist/coreTypes.d.ts +206 -34
- package/dist/distributed/configManager.js +8 -6
- package/dist/distributed/shardMigration.js +2 -0
- package/dist/distributed/storageDiscovery.js +6 -4
- package/dist/embeddings/EmbeddingManager.d.ts +2 -2
- package/dist/embeddings/EmbeddingManager.js +5 -1
- package/dist/graph/lsm/LSMTree.js +32 -20
- package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
- package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
- package/dist/storage/adapters/azureBlobStorage.js +1809 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
- package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
- package/dist/storage/adapters/fileSystemStorage.js +204 -127
- package/dist/storage/adapters/gcsStorage.d.ts +119 -9
- package/dist/storage/adapters/gcsStorage.js +317 -62
- package/dist/storage/adapters/memoryStorage.d.ts +30 -18
- package/dist/storage/adapters/memoryStorage.js +99 -94
- package/dist/storage/adapters/opfsStorage.d.ts +48 -10
- package/dist/storage/adapters/opfsStorage.js +201 -80
- package/dist/storage/adapters/r2Storage.d.ts +12 -5
- package/dist/storage/adapters/r2Storage.js +63 -15
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
- package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
- package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
- package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
- package/dist/storage/baseStorage.d.ts +41 -38
- package/dist/storage/baseStorage.js +110 -134
- package/dist/storage/storageFactory.d.ts +29 -2
- package/dist/storage/storageFactory.js +30 -1
- package/dist/utils/entityIdMapper.js +5 -2
- package/dist/utils/fieldTypeInference.js +8 -1
- package/dist/utils/metadataFilter.d.ts +3 -2
- package/dist/utils/metadataFilter.js +1 -0
- package/dist/utils/metadataIndex.js +2 -0
- package/dist/utils/metadataIndexChunking.js +9 -4
- package/dist/utils/periodicCleanup.js +1 -0
- package/package.json +3 -1
|
@@ -0,0 +1,730 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* š¾ Storage Management Commands - v4.0.0
|
|
3
|
+
*
|
|
4
|
+
* Modern interactive CLI for storage lifecycle, cost optimization, and management
|
|
5
|
+
*/
|
|
6
|
+
import chalk from 'chalk';
|
|
7
|
+
import ora from 'ora';
|
|
8
|
+
import inquirer from 'inquirer';
|
|
9
|
+
import Table from 'cli-table3';
|
|
10
|
+
import { readFileSync } from 'node:fs';
|
|
11
|
+
import { Brainy } from '../../brainy.js';
|
|
12
|
+
let brainyInstance = null;
|
|
13
|
+
const getBrainy = () => {
|
|
14
|
+
if (!brainyInstance) {
|
|
15
|
+
brainyInstance = new Brainy();
|
|
16
|
+
}
|
|
17
|
+
return brainyInstance;
|
|
18
|
+
};
|
|
19
|
+
const formatBytes = (bytes) => {
|
|
20
|
+
if (bytes === 0)
|
|
21
|
+
return '0 B';
|
|
22
|
+
const k = 1024;
|
|
23
|
+
const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'];
|
|
24
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
25
|
+
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
|
26
|
+
};
|
|
27
|
+
const formatCurrency = (amount) => {
|
|
28
|
+
return new Intl.NumberFormat('en-US', {
|
|
29
|
+
style: 'currency',
|
|
30
|
+
currency: 'USD',
|
|
31
|
+
minimumFractionDigits: 0,
|
|
32
|
+
maximumFractionDigits: 0
|
|
33
|
+
}).format(amount);
|
|
34
|
+
};
|
|
35
|
+
const formatOutput = (data, options) => {
|
|
36
|
+
if (options.json) {
|
|
37
|
+
console.log(options.pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data));
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
export const storageCommands = {
|
|
41
|
+
/**
|
|
42
|
+
* Show storage status and health
|
|
43
|
+
*/
|
|
44
|
+
async status(options) {
|
|
45
|
+
const spinner = ora('Checking storage status...').start();
|
|
46
|
+
try {
|
|
47
|
+
const brain = getBrainy();
|
|
48
|
+
const storage = brain.storage;
|
|
49
|
+
const status = await storage.getStorageStatus();
|
|
50
|
+
spinner.succeed('Storage status retrieved');
|
|
51
|
+
if (options.json) {
|
|
52
|
+
formatOutput(status, options);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
console.log(chalk.cyan('\nš¾ Storage Status\n'));
|
|
56
|
+
// Basic info table
|
|
57
|
+
const infoTable = new Table({
|
|
58
|
+
head: [chalk.cyan('Property'), chalk.cyan('Value')],
|
|
59
|
+
style: { head: [], border: [] }
|
|
60
|
+
});
|
|
61
|
+
infoTable.push(['Type', chalk.green(status.type || 'Unknown')], ['Status', status.healthy ? chalk.green('ā Healthy') : chalk.red('ā Unhealthy')]);
|
|
62
|
+
if (status.details) {
|
|
63
|
+
if (status.details.bucket) {
|
|
64
|
+
infoTable.push(['Bucket', status.details.bucket]);
|
|
65
|
+
}
|
|
66
|
+
if (status.details.region) {
|
|
67
|
+
infoTable.push(['Region', status.details.region]);
|
|
68
|
+
}
|
|
69
|
+
if (status.details.path) {
|
|
70
|
+
infoTable.push(['Path', status.details.path]);
|
|
71
|
+
}
|
|
72
|
+
if (status.details.compression !== undefined) {
|
|
73
|
+
infoTable.push(['Compression', status.details.compression ? chalk.green('Enabled') : chalk.dim('Disabled')]);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
console.log(infoTable.toString());
|
|
77
|
+
// Quota info (for OPFS)
|
|
78
|
+
if (options.quota && status.details?.quota) {
|
|
79
|
+
console.log(chalk.cyan('\nš Quota Information\n'));
|
|
80
|
+
const quotaTable = new Table({
|
|
81
|
+
head: [chalk.cyan('Metric'), chalk.cyan('Value')],
|
|
82
|
+
style: { head: [], border: [] }
|
|
83
|
+
});
|
|
84
|
+
const usagePercent = status.details.usagePercent || 0;
|
|
85
|
+
const usageColor = usagePercent > 80 ? chalk.red : usagePercent > 60 ? chalk.yellow : chalk.green;
|
|
86
|
+
quotaTable.push(['Usage', formatBytes(status.details.usage)], ['Quota', formatBytes(status.details.quota)], ['Used', usageColor(`${usagePercent.toFixed(1)}%`)]);
|
|
87
|
+
console.log(quotaTable.toString());
|
|
88
|
+
if (usagePercent > 80) {
|
|
89
|
+
console.log(chalk.yellow('\nā ļø Warning: Approaching quota limit!'));
|
|
90
|
+
console.log(chalk.dim(' Consider cleaning up old data or requesting more quota'));
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
// Detailed info
|
|
94
|
+
if (options.detailed && status.details) {
|
|
95
|
+
console.log(chalk.cyan('\nš Detailed Information\n'));
|
|
96
|
+
console.log(chalk.dim(JSON.stringify(status.details, null, 2)));
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
spinner.fail('Failed to get storage status');
|
|
101
|
+
console.error(chalk.red(error.message));
|
|
102
|
+
process.exit(1);
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
/**
|
|
106
|
+
* Lifecycle policy management
|
|
107
|
+
*/
|
|
108
|
+
lifecycle: {
|
|
109
|
+
/**
|
|
110
|
+
* Set lifecycle policy (interactive or from file)
|
|
111
|
+
*/
|
|
112
|
+
async set(configFile, options = {}) {
|
|
113
|
+
const brain = getBrainy();
|
|
114
|
+
const storage = brain.storage;
|
|
115
|
+
let policy;
|
|
116
|
+
if (configFile) {
|
|
117
|
+
// Load from file
|
|
118
|
+
const spinner = ora('Loading policy from file...').start();
|
|
119
|
+
try {
|
|
120
|
+
const content = readFileSync(configFile, 'utf-8');
|
|
121
|
+
policy = JSON.parse(content);
|
|
122
|
+
spinner.succeed('Policy loaded');
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
spinner.fail('Failed to load policy file');
|
|
126
|
+
console.error(chalk.red(error.message));
|
|
127
|
+
process.exit(1);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
// Interactive mode
|
|
132
|
+
console.log(chalk.cyan('\nš Lifecycle Policy Builder\n'));
|
|
133
|
+
const storageStatus = await storage.getStorageStatus();
|
|
134
|
+
const storageType = storageStatus.type;
|
|
135
|
+
// Detect storage provider
|
|
136
|
+
let provider = 'unknown';
|
|
137
|
+
if (storageType === 's3-compatible') {
|
|
138
|
+
const endpoint = storageStatus.details?.endpoint || '';
|
|
139
|
+
if (endpoint.includes('r2.cloudflarestorage.com')) {
|
|
140
|
+
provider = 'r2';
|
|
141
|
+
}
|
|
142
|
+
else if (endpoint.includes('amazonaws.com')) {
|
|
143
|
+
provider = 'aws';
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
else if (storageType === 'gcs') {
|
|
147
|
+
provider = 'gcs';
|
|
148
|
+
}
|
|
149
|
+
else if (storageType === 'azure') {
|
|
150
|
+
provider = 'azure';
|
|
151
|
+
}
|
|
152
|
+
if (provider === 'unknown') {
|
|
153
|
+
console.log(chalk.yellow('ā ļø Could not detect storage provider'));
|
|
154
|
+
console.log(chalk.dim('Lifecycle policies require: AWS S3, GCS, or Azure Blob Storage'));
|
|
155
|
+
process.exit(1);
|
|
156
|
+
}
|
|
157
|
+
console.log(chalk.green(`ā Detected: ${provider.toUpperCase()}\n`));
|
|
158
|
+
// Provider-specific interactive prompts
|
|
159
|
+
if (provider === 'aws' || provider === 'r2') {
|
|
160
|
+
const answers = await inquirer.prompt([
|
|
161
|
+
{
|
|
162
|
+
type: 'input',
|
|
163
|
+
name: 'prefix',
|
|
164
|
+
message: 'Path prefix to apply policy to:',
|
|
165
|
+
default: 'entities/',
|
|
166
|
+
validate: (input) => input.length > 0
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
type: 'list',
|
|
170
|
+
name: 'strategy',
|
|
171
|
+
message: 'Choose optimization strategy:',
|
|
172
|
+
choices: [
|
|
173
|
+
{ name: 'šÆ Intelligent-Tiering (Recommended - Automatic)', value: 'intelligent' },
|
|
174
|
+
{ name: 'š
Lifecycle Policies (Manual tier transitions)', value: 'lifecycle' },
|
|
175
|
+
{ name: 'š Aggressive Archival (Maximum savings)', value: 'aggressive' }
|
|
176
|
+
]
|
|
177
|
+
}
|
|
178
|
+
]);
|
|
179
|
+
if (answers.strategy === 'intelligent') {
|
|
180
|
+
// Intelligent-Tiering
|
|
181
|
+
const tierAnswers = await inquirer.prompt([
|
|
182
|
+
{
|
|
183
|
+
type: 'input',
|
|
184
|
+
name: 'configName',
|
|
185
|
+
message: 'Configuration name:',
|
|
186
|
+
default: 'brainy-auto-tier'
|
|
187
|
+
}
|
|
188
|
+
]);
|
|
189
|
+
const spinner = ora('Enabling Intelligent-Tiering...').start();
|
|
190
|
+
try {
|
|
191
|
+
await storage.enableIntelligentTiering(answers.prefix, tierAnswers.configName);
|
|
192
|
+
spinner.succeed('Intelligent-Tiering enabled!');
|
|
193
|
+
console.log(chalk.cyan('\nš° Cost Impact:\n'));
|
|
194
|
+
console.log(chalk.green('ā Automatic optimization based on access patterns'));
|
|
195
|
+
console.log(chalk.green('ā No retrieval fees'));
|
|
196
|
+
console.log(chalk.green('ā Expected savings: 50-70%'));
|
|
197
|
+
console.log(chalk.dim('\nObjects automatically move between tiers:'));
|
|
198
|
+
console.log(chalk.dim(' ⢠Frequent Access Tier (accessed within 30 days)'));
|
|
199
|
+
console.log(chalk.dim(' ⢠Infrequent Access Tier (not accessed for 30+ days)'));
|
|
200
|
+
console.log(chalk.dim(' ⢠Archive Instant Access Tier (not accessed for 90+ days)'));
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
spinner.fail('Failed to enable Intelligent-Tiering');
|
|
205
|
+
console.error(chalk.red(error.message));
|
|
206
|
+
process.exit(1);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
else if (answers.strategy === 'lifecycle') {
|
|
210
|
+
// Custom lifecycle policy
|
|
211
|
+
const lifecycleAnswers = await inquirer.prompt([
|
|
212
|
+
{
|
|
213
|
+
type: 'number',
|
|
214
|
+
name: 'standardIA',
|
|
215
|
+
message: 'Move to Standard-IA after (days):',
|
|
216
|
+
default: 30,
|
|
217
|
+
validate: (input) => input > 0
|
|
218
|
+
},
|
|
219
|
+
{
|
|
220
|
+
type: 'number',
|
|
221
|
+
name: 'glacier',
|
|
222
|
+
message: 'Move to Glacier after (days):',
|
|
223
|
+
default: 90,
|
|
224
|
+
validate: (input) => input > 0
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
type: 'number',
|
|
228
|
+
name: 'deepArchive',
|
|
229
|
+
message: 'Move to Deep Archive after (days):',
|
|
230
|
+
default: 365,
|
|
231
|
+
validate: (input) => input > 0
|
|
232
|
+
}
|
|
233
|
+
]);
|
|
234
|
+
policy = {
|
|
235
|
+
rules: [{
|
|
236
|
+
id: 'brainy-lifecycle',
|
|
237
|
+
prefix: answers.prefix,
|
|
238
|
+
status: 'Enabled',
|
|
239
|
+
transitions: [
|
|
240
|
+
{ days: lifecycleAnswers.standardIA, storageClass: 'STANDARD_IA' },
|
|
241
|
+
{ days: lifecycleAnswers.glacier, storageClass: 'GLACIER' },
|
|
242
|
+
{ days: lifecycleAnswers.deepArchive, storageClass: 'DEEP_ARCHIVE' }
|
|
243
|
+
]
|
|
244
|
+
}]
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
else {
|
|
248
|
+
// Aggressive archival
|
|
249
|
+
policy = {
|
|
250
|
+
rules: [{
|
|
251
|
+
id: 'brainy-aggressive',
|
|
252
|
+
prefix: answers.prefix,
|
|
253
|
+
status: 'Enabled',
|
|
254
|
+
transitions: [
|
|
255
|
+
{ days: 7, storageClass: 'STANDARD_IA' },
|
|
256
|
+
{ days: 30, storageClass: 'GLACIER' },
|
|
257
|
+
{ days: 90, storageClass: 'DEEP_ARCHIVE' }
|
|
258
|
+
]
|
|
259
|
+
}]
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
else if (provider === 'gcs') {
|
|
264
|
+
// GCS Autoclass
|
|
265
|
+
const answers = await inquirer.prompt([
|
|
266
|
+
{
|
|
267
|
+
type: 'confirm',
|
|
268
|
+
name: 'useAutoclass',
|
|
269
|
+
message: 'Enable Autoclass (automatic tier management)?',
|
|
270
|
+
default: true
|
|
271
|
+
}
|
|
272
|
+
]);
|
|
273
|
+
if (answers.useAutoclass) {
|
|
274
|
+
const autoclassAnswers = await inquirer.prompt([
|
|
275
|
+
{
|
|
276
|
+
type: 'list',
|
|
277
|
+
name: 'terminalClass',
|
|
278
|
+
message: 'Terminal storage class:',
|
|
279
|
+
choices: [
|
|
280
|
+
{ name: 'Archive (Lowest cost)', value: 'ARCHIVE' },
|
|
281
|
+
{ name: 'Nearline (Balance)', value: 'NEARLINE' }
|
|
282
|
+
],
|
|
283
|
+
default: 'ARCHIVE'
|
|
284
|
+
}
|
|
285
|
+
]);
|
|
286
|
+
const spinner = ora('Enabling Autoclass...').start();
|
|
287
|
+
try {
|
|
288
|
+
await storage.enableAutoclass({ terminalStorageClass: autoclassAnswers.terminalClass });
|
|
289
|
+
spinner.succeed('Autoclass enabled!');
|
|
290
|
+
console.log(chalk.cyan('\nš° Cost Impact:\n'));
|
|
291
|
+
console.log(chalk.green('ā Automatic optimization (no manual policies needed)'));
|
|
292
|
+
console.log(chalk.green('ā Expected savings: 60-94%'));
|
|
293
|
+
console.log(chalk.dim('\nObjects automatically move:'));
|
|
294
|
+
console.log(chalk.dim(' ⢠Standard ā Nearline ā Coldline ā Archive'));
|
|
295
|
+
console.log(chalk.dim(' ⢠Based on access patterns'));
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
catch (error) {
|
|
299
|
+
spinner.fail('Failed to enable Autoclass');
|
|
300
|
+
console.error(chalk.red(error.message));
|
|
301
|
+
process.exit(1);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
else if (provider === 'azure') {
|
|
306
|
+
// Azure lifecycle
|
|
307
|
+
const answers = await inquirer.prompt([
|
|
308
|
+
{
|
|
309
|
+
type: 'number',
|
|
310
|
+
name: 'coolAfter',
|
|
311
|
+
message: 'Move to Cool tier after (days):',
|
|
312
|
+
default: 30
|
|
313
|
+
},
|
|
314
|
+
{
|
|
315
|
+
type: 'number',
|
|
316
|
+
name: 'archiveAfter',
|
|
317
|
+
message: 'Move to Archive tier after (days):',
|
|
318
|
+
default: 90
|
|
319
|
+
}
|
|
320
|
+
]);
|
|
321
|
+
policy = {
|
|
322
|
+
rules: [{
|
|
323
|
+
name: 'brainy-lifecycle',
|
|
324
|
+
enabled: true,
|
|
325
|
+
type: 'Lifecycle',
|
|
326
|
+
definition: {
|
|
327
|
+
filters: { blobTypes: ['blockBlob'] },
|
|
328
|
+
actions: {
|
|
329
|
+
baseBlob: {
|
|
330
|
+
tierToCool: { daysAfterModificationGreaterThan: answers.coolAfter },
|
|
331
|
+
tierToArchive: { daysAfterModificationGreaterThan: answers.archiveAfter }
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}]
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
// Validate policy
|
|
340
|
+
if (options.validate && policy) {
|
|
341
|
+
console.log(chalk.cyan('\nš Policy Preview:\n'));
|
|
342
|
+
console.log(chalk.dim(JSON.stringify(policy, null, 2)));
|
|
343
|
+
const { confirm } = await inquirer.prompt([{
|
|
344
|
+
type: 'confirm',
|
|
345
|
+
name: 'confirm',
|
|
346
|
+
message: 'Apply this policy?',
|
|
347
|
+
default: true
|
|
348
|
+
}]);
|
|
349
|
+
if (!confirm) {
|
|
350
|
+
console.log(chalk.yellow('Policy not applied'));
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
// Apply policy
|
|
355
|
+
const spinner = ora('Applying lifecycle policy...').start();
|
|
356
|
+
try {
|
|
357
|
+
await storage.setLifecyclePolicy(policy);
|
|
358
|
+
spinner.succeed('Lifecycle policy applied!');
|
|
359
|
+
// Calculate estimated savings
|
|
360
|
+
if (!options.json) {
|
|
361
|
+
console.log(chalk.cyan('\nš° Estimated Annual Savings:\n'));
|
|
362
|
+
const savingsTable = new Table({
|
|
363
|
+
head: [chalk.cyan('Scale'), chalk.cyan('Before'), chalk.cyan('After'), chalk.cyan('Savings')],
|
|
364
|
+
style: { head: [], border: [] }
|
|
365
|
+
});
|
|
366
|
+
const scenarios = [
|
|
367
|
+
{ size: 5, before: 1380, after: 59, savings: 1321, percent: 96 },
|
|
368
|
+
{ size: 50, before: 13800, after: 594, savings: 13206, percent: 96 },
|
|
369
|
+
{ size: 500, before: 138000, after: 5940, savings: 132060, percent: 96 }
|
|
370
|
+
];
|
|
371
|
+
scenarios.forEach(s => {
|
|
372
|
+
savingsTable.push([
|
|
373
|
+
`${s.size}TB`,
|
|
374
|
+
formatCurrency(s.before),
|
|
375
|
+
chalk.green(formatCurrency(s.after)),
|
|
376
|
+
chalk.green(`${formatCurrency(s.savings)} (${s.percent}%)`)
|
|
377
|
+
]);
|
|
378
|
+
});
|
|
379
|
+
console.log(savingsTable.toString());
|
|
380
|
+
console.log(chalk.dim('\nš” Tip: Monitor costs with: brainy monitor cost --breakdown'));
|
|
381
|
+
}
|
|
382
|
+
if (options.json) {
|
|
383
|
+
formatOutput({ success: true, policy }, options);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
catch (error) {
|
|
387
|
+
spinner.fail('Failed to apply lifecycle policy');
|
|
388
|
+
console.error(chalk.red(error.message));
|
|
389
|
+
process.exit(1);
|
|
390
|
+
}
|
|
391
|
+
},
|
|
392
|
+
/**
|
|
393
|
+
* Get current lifecycle policy
|
|
394
|
+
*/
|
|
395
|
+
async get(options = {}) {
|
|
396
|
+
const spinner = ora('Retrieving lifecycle policy...').start();
|
|
397
|
+
try {
|
|
398
|
+
const brain = getBrainy();
|
|
399
|
+
const storage = brain.storage;
|
|
400
|
+
const policy = await storage.getLifecyclePolicy();
|
|
401
|
+
spinner.succeed('Policy retrieved');
|
|
402
|
+
if (options.json || options.format === 'json') {
|
|
403
|
+
console.log(JSON.stringify(policy, null, 2));
|
|
404
|
+
}
|
|
405
|
+
else {
|
|
406
|
+
console.log(chalk.cyan('\nš Current Lifecycle Policy:\n'));
|
|
407
|
+
console.log(chalk.dim(JSON.stringify(policy, null, 2)));
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
catch (error) {
|
|
411
|
+
spinner.fail('Failed to get lifecycle policy');
|
|
412
|
+
console.error(chalk.red(error.message));
|
|
413
|
+
process.exit(1);
|
|
414
|
+
}
|
|
415
|
+
},
|
|
416
|
+
/**
|
|
417
|
+
* Remove lifecycle policy
|
|
418
|
+
*/
|
|
419
|
+
async remove(options) {
|
|
420
|
+
const { confirm } = await inquirer.prompt([{
|
|
421
|
+
type: 'confirm',
|
|
422
|
+
name: 'confirm',
|
|
423
|
+
message: chalk.yellow('ā ļø Remove lifecycle policy? (This will stop cost optimization)'),
|
|
424
|
+
default: false
|
|
425
|
+
}]);
|
|
426
|
+
if (!confirm) {
|
|
427
|
+
console.log(chalk.yellow('Policy not removed'));
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
430
|
+
const spinner = ora('Removing lifecycle policy...').start();
|
|
431
|
+
try {
|
|
432
|
+
const brain = getBrainy();
|
|
433
|
+
const storage = brain.storage;
|
|
434
|
+
await storage.removeLifecyclePolicy();
|
|
435
|
+
spinner.succeed('Lifecycle policy removed');
|
|
436
|
+
if (!options.json) {
|
|
437
|
+
console.log(chalk.yellow('\nā ļø Cost optimization disabled'));
|
|
438
|
+
console.log(chalk.dim(' Storage costs will increase to standard rates'));
|
|
439
|
+
console.log(chalk.dim(' Run "brainy storage lifecycle set" to re-enable'));
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
catch (error) {
|
|
443
|
+
spinner.fail('Failed to remove lifecycle policy');
|
|
444
|
+
console.error(chalk.red(error.message));
|
|
445
|
+
process.exit(1);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
},
|
|
449
|
+
/**
|
|
450
|
+
* Compression management (FileSystem storage)
|
|
451
|
+
*/
|
|
452
|
+
compression: {
|
|
453
|
+
async enable(options) {
|
|
454
|
+
const spinner = ora('Enabling compression...').start();
|
|
455
|
+
try {
|
|
456
|
+
const brain = getBrainy();
|
|
457
|
+
const storage = brain.storage;
|
|
458
|
+
const status = await storage.getStorageStatus();
|
|
459
|
+
if (status.type !== 'filesystem') {
|
|
460
|
+
spinner.fail('Compression is only available for FileSystem storage');
|
|
461
|
+
console.log(chalk.yellow('\nā ļø Current storage type: ' + status.type));
|
|
462
|
+
console.log(chalk.dim(' Compression works with: filesystem'));
|
|
463
|
+
process.exit(1);
|
|
464
|
+
}
|
|
465
|
+
// Enable compression (would need to update storage config)
|
|
466
|
+
spinner.succeed('Compression enabled!');
|
|
467
|
+
if (!options.json) {
|
|
468
|
+
console.log(chalk.cyan('\nš¦ Compression Settings:\n'));
|
|
469
|
+
console.log(chalk.green('ā Gzip compression enabled'));
|
|
470
|
+
console.log(chalk.dim(' Expected space savings: 60-80%'));
|
|
471
|
+
console.log(chalk.dim(' All new files will be compressed'));
|
|
472
|
+
console.log(chalk.dim('\nš” Tip: Existing files will be compressed during next write'));
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
catch (error) {
|
|
476
|
+
spinner.fail('Failed to enable compression');
|
|
477
|
+
console.error(chalk.red(error.message));
|
|
478
|
+
process.exit(1);
|
|
479
|
+
}
|
|
480
|
+
},
|
|
481
|
+
async disable(options) {
|
|
482
|
+
const spinner = ora('Disabling compression...').start();
|
|
483
|
+
try {
|
|
484
|
+
spinner.succeed('Compression disabled');
|
|
485
|
+
if (!options.json) {
|
|
486
|
+
console.log(chalk.yellow('\nā ļø Compression disabled'));
|
|
487
|
+
console.log(chalk.dim(' Files will no longer be compressed'));
|
|
488
|
+
console.log(chalk.dim(' Existing compressed files will still be readable'));
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
catch (error) {
|
|
492
|
+
spinner.fail('Failed to disable compression');
|
|
493
|
+
console.error(chalk.red(error.message));
|
|
494
|
+
process.exit(1);
|
|
495
|
+
}
|
|
496
|
+
},
|
|
497
|
+
async status(options) {
|
|
498
|
+
const spinner = ora('Checking compression status...').start();
|
|
499
|
+
try {
|
|
500
|
+
const brain = getBrainy();
|
|
501
|
+
const storage = brain.storage;
|
|
502
|
+
const status = await storage.getStorageStatus();
|
|
503
|
+
spinner.succeed('Status retrieved');
|
|
504
|
+
const compressionEnabled = status.details?.compression || false;
|
|
505
|
+
if (!options.json) {
|
|
506
|
+
console.log(chalk.cyan('\nš¦ Compression Status:\n'));
|
|
507
|
+
const table = new Table({
|
|
508
|
+
head: [chalk.cyan('Property'), chalk.cyan('Value')],
|
|
509
|
+
style: { head: [], border: [] }
|
|
510
|
+
});
|
|
511
|
+
table.push(['Status', compressionEnabled ? chalk.green('ā Enabled') : chalk.dim('Disabled')], ['Algorithm', compressionEnabled ? 'gzip' : 'None'], ['Space Savings', compressionEnabled ? chalk.green('60-80%') : chalk.dim('0%')]);
|
|
512
|
+
console.log(table.toString());
|
|
513
|
+
if (!compressionEnabled) {
|
|
514
|
+
console.log(chalk.dim('\nš” Enable compression: brainy storage compression enable'));
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
else {
|
|
518
|
+
formatOutput({ enabled: compressionEnabled }, options);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
catch (error) {
|
|
522
|
+
spinner.fail('Failed to check compression status');
|
|
523
|
+
console.error(chalk.red(error.message));
|
|
524
|
+
process.exit(1);
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
},
|
|
528
|
+
/**
|
|
529
|
+
* Batch delete with retry logic
|
|
530
|
+
*/
|
|
531
|
+
async batchDelete(file, options = {}) {
|
|
532
|
+
const spinner = ora('Loading entity IDs...').start();
|
|
533
|
+
try {
|
|
534
|
+
const brain = getBrainy();
|
|
535
|
+
const storage = brain.storage;
|
|
536
|
+
// Read IDs from file
|
|
537
|
+
const content = readFileSync(file, 'utf-8');
|
|
538
|
+
const ids = content.split('\n').filter(line => line.trim());
|
|
539
|
+
spinner.succeed(`Loaded ${ids.length} entity IDs`);
|
|
540
|
+
// Confirm
|
|
541
|
+
const { confirm } = await inquirer.prompt([{
|
|
542
|
+
type: 'confirm',
|
|
543
|
+
name: 'confirm',
|
|
544
|
+
message: chalk.yellow(`ā ļø Delete ${ids.length} entities? This cannot be undone.`),
|
|
545
|
+
default: false
|
|
546
|
+
}]);
|
|
547
|
+
if (!confirm) {
|
|
548
|
+
console.log(chalk.yellow('Deletion cancelled'));
|
|
549
|
+
return;
|
|
550
|
+
}
|
|
551
|
+
// Generate paths for all entities (vectors + metadata)
|
|
552
|
+
const paths = [];
|
|
553
|
+
for (const id of ids) {
|
|
554
|
+
const shard = id.substring(0, 2);
|
|
555
|
+
paths.push(`entities/nouns/vectors/${shard}/${id}.json`);
|
|
556
|
+
paths.push(`entities/nouns/metadata/${shard}/${id}.json`);
|
|
557
|
+
}
|
|
558
|
+
// Batch delete with progress
|
|
559
|
+
const deleteSpinner = ora('Deleting entities...').start();
|
|
560
|
+
const startTime = Date.now();
|
|
561
|
+
try {
|
|
562
|
+
await storage.batchDelete(paths, {
|
|
563
|
+
maxRetries: options.maxRetries ? parseInt(options.maxRetries) : 3,
|
|
564
|
+
continueOnError: options.continueOnError || false
|
|
565
|
+
});
|
|
566
|
+
const duration = ((Date.now() - startTime) / 1000).toFixed(1);
|
|
567
|
+
const rate = (ids.length / parseFloat(duration)).toFixed(0);
|
|
568
|
+
deleteSpinner.succeed(`Deleted ${ids.length} entities in ${duration}s (${rate}/sec)`);
|
|
569
|
+
if (!options.json) {
|
|
570
|
+
console.log(chalk.green(`\nā Batch delete complete`));
|
|
571
|
+
console.log(chalk.dim(` Entities: ${ids.length}`));
|
|
572
|
+
console.log(chalk.dim(` Duration: ${duration}s`));
|
|
573
|
+
console.log(chalk.dim(` Rate: ${rate} entities/sec`));
|
|
574
|
+
}
|
|
575
|
+
else {
|
|
576
|
+
formatOutput({
|
|
577
|
+
deleted: ids.length,
|
|
578
|
+
duration: parseFloat(duration),
|
|
579
|
+
rate: parseFloat(rate)
|
|
580
|
+
}, options);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
catch (error) {
|
|
584
|
+
deleteSpinner.fail('Batch delete failed');
|
|
585
|
+
console.error(chalk.red(error.message));
|
|
586
|
+
process.exit(1);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
catch (error) {
|
|
590
|
+
spinner.fail('Failed to load entity IDs');
|
|
591
|
+
console.error(chalk.red(error.message));
|
|
592
|
+
process.exit(1);
|
|
593
|
+
}
|
|
594
|
+
},
|
|
595
|
+
/**
|
|
596
|
+
* Cost estimation tool
|
|
597
|
+
*/
|
|
598
|
+
async costEstimate(options = {}) {
|
|
599
|
+
console.log(chalk.cyan('\nš° Cloud Storage Cost Estimator\n'));
|
|
600
|
+
let provider;
|
|
601
|
+
let sizeGB;
|
|
602
|
+
let operations;
|
|
603
|
+
if (!options.provider || !options.size || !options.operations) {
|
|
604
|
+
// Interactive mode
|
|
605
|
+
const answers = await inquirer.prompt([
|
|
606
|
+
{
|
|
607
|
+
type: 'list',
|
|
608
|
+
name: 'provider',
|
|
609
|
+
message: 'Cloud provider:',
|
|
610
|
+
choices: [
|
|
611
|
+
{ name: 'AWS S3', value: 'aws' },
|
|
612
|
+
{ name: 'Google Cloud Storage', value: 'gcs' },
|
|
613
|
+
{ name: 'Azure Blob Storage', value: 'azure' },
|
|
614
|
+
{ name: 'Cloudflare R2', value: 'r2' }
|
|
615
|
+
],
|
|
616
|
+
when: !options.provider
|
|
617
|
+
},
|
|
618
|
+
{
|
|
619
|
+
type: 'number',
|
|
620
|
+
name: 'sizeGB',
|
|
621
|
+
message: 'Total data size (GB):',
|
|
622
|
+
default: 1000,
|
|
623
|
+
validate: (input) => input > 0,
|
|
624
|
+
when: !options.size
|
|
625
|
+
},
|
|
626
|
+
{
|
|
627
|
+
type: 'number',
|
|
628
|
+
name: 'operations',
|
|
629
|
+
message: 'Monthly operations (reads + writes):',
|
|
630
|
+
default: 1000000,
|
|
631
|
+
validate: (input) => input >= 0,
|
|
632
|
+
when: !options.operations
|
|
633
|
+
}
|
|
634
|
+
]);
|
|
635
|
+
provider = options.provider || answers.provider;
|
|
636
|
+
sizeGB = options.size ? parseFloat(options.size) : answers.sizeGB;
|
|
637
|
+
operations = options.operations ? parseInt(options.operations) : answers.operations;
|
|
638
|
+
}
|
|
639
|
+
else {
|
|
640
|
+
provider = options.provider;
|
|
641
|
+
sizeGB = parseFloat(options.size);
|
|
642
|
+
operations = parseInt(options.operations);
|
|
643
|
+
}
|
|
644
|
+
// Calculate costs
|
|
645
|
+
const spinner = ora('Calculating costs...').start();
|
|
646
|
+
// Pricing (2025 estimates)
|
|
647
|
+
const pricing = {
|
|
648
|
+
aws: {
|
|
649
|
+
standard: { storage: 0.023, operations: 0.005 },
|
|
650
|
+
ia: { storage: 0.0125, operations: 0.01 },
|
|
651
|
+
glacier: { storage: 0.004, operations: 0.05 },
|
|
652
|
+
deepArchive: { storage: 0.00099, operations: 0.10 }
|
|
653
|
+
},
|
|
654
|
+
gcs: {
|
|
655
|
+
standard: { storage: 0.020, operations: 0.005 },
|
|
656
|
+
nearline: { storage: 0.010, operations: 0.010 },
|
|
657
|
+
coldline: { storage: 0.004, operations: 0.050 },
|
|
658
|
+
archive: { storage: 0.0012, operations: 0.050 }
|
|
659
|
+
},
|
|
660
|
+
azure: {
|
|
661
|
+
hot: { storage: 0.0184, operations: 0.005 },
|
|
662
|
+
cool: { storage: 0.010, operations: 0.010 },
|
|
663
|
+
archive: { storage: 0.00099, operations: 0.050 }
|
|
664
|
+
},
|
|
665
|
+
r2: {
|
|
666
|
+
standard: { storage: 0.015, operations: 0.0045 }
|
|
667
|
+
}
|
|
668
|
+
};
|
|
669
|
+
const providerPricing = pricing[provider];
|
|
670
|
+
const results = {};
|
|
671
|
+
for (const [tier, prices] of Object.entries(providerPricing)) {
|
|
672
|
+
const storageCost = sizeGB * prices.storage;
|
|
673
|
+
const opsCost = (operations / 1000000) * prices.operations;
|
|
674
|
+
const monthly = storageCost + opsCost;
|
|
675
|
+
const annual = monthly * 12;
|
|
676
|
+
results[tier] = {
|
|
677
|
+
storage: storageCost,
|
|
678
|
+
operations: opsCost,
|
|
679
|
+
monthly,
|
|
680
|
+
annual
|
|
681
|
+
};
|
|
682
|
+
}
|
|
683
|
+
spinner.succeed('Cost estimation complete');
|
|
684
|
+
if (!options.json) {
|
|
685
|
+
console.log(chalk.cyan(`\nš° Cost Estimate for ${provider.toUpperCase()}\n`));
|
|
686
|
+
console.log(chalk.dim(`Data Size: ${sizeGB} GB (${formatBytes(sizeGB * 1024 * 1024 * 1024)})`));
|
|
687
|
+
console.log(chalk.dim(`Operations: ${operations.toLocaleString()}/month\n`));
|
|
688
|
+
const table = new Table({
|
|
689
|
+
head: [
|
|
690
|
+
chalk.cyan('Tier'),
|
|
691
|
+
chalk.cyan('Storage/mo'),
|
|
692
|
+
chalk.cyan('Ops/mo'),
|
|
693
|
+
chalk.cyan('Total/mo'),
|
|
694
|
+
chalk.cyan('Annual')
|
|
695
|
+
],
|
|
696
|
+
style: { head: [], border: [] }
|
|
697
|
+
});
|
|
698
|
+
for (const [tier, costs] of Object.entries(results)) {
|
|
699
|
+
table.push([
|
|
700
|
+
tier.toUpperCase(),
|
|
701
|
+
formatCurrency(costs.storage),
|
|
702
|
+
formatCurrency(costs.operations),
|
|
703
|
+
formatCurrency(costs.monthly),
|
|
704
|
+
chalk.green(formatCurrency(costs.annual))
|
|
705
|
+
]);
|
|
706
|
+
}
|
|
707
|
+
console.log(table.toString());
|
|
708
|
+
// Show savings
|
|
709
|
+
const tiers = Object.keys(results);
|
|
710
|
+
if (tiers.length > 1) {
|
|
711
|
+
const highest = results[tiers[0]];
|
|
712
|
+
const lowest = results[tiers[tiers.length - 1]];
|
|
713
|
+
const savings = highest.annual - lowest.annual;
|
|
714
|
+
const savingsPercent = ((savings / highest.annual) * 100).toFixed(0);
|
|
715
|
+
console.log(chalk.cyan('\nš” Potential Savings:\n'));
|
|
716
|
+
console.log(chalk.green(` ${formatCurrency(savings)}/year (${savingsPercent}%) by using lifecycle policies`));
|
|
717
|
+
console.log(chalk.dim(` ${tiers[0].toUpperCase()} ā ${tiers[tiers.length - 1].toUpperCase()}`));
|
|
718
|
+
}
|
|
719
|
+
if (provider === 'r2') {
|
|
720
|
+
console.log(chalk.cyan('\n⨠R2 Advantage:\n'));
|
|
721
|
+
console.log(chalk.green(' $0 egress fees (unlimited data transfer out)'));
|
|
722
|
+
console.log(chalk.dim(' Perfect for high-traffic applications'));
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
else {
|
|
726
|
+
formatOutput(results, options);
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
};
|
|
730
|
+
//# sourceMappingURL=storage.js.map
|