@soulcraft/brainy 5.10.4 → 5.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brainy.d.ts +89 -0
- package/dist/brainy.js +154 -1
- package/dist/storage/adapters/azureBlobStorage.d.ts +2 -5
- package/dist/storage/adapters/azureBlobStorage.js +4 -40
- package/dist/storage/adapters/fileSystemStorage.d.ts +2 -5
- package/dist/storage/adapters/fileSystemStorage.js +4 -42
- package/dist/storage/adapters/gcsStorage.d.ts +2 -5
- package/dist/storage/adapters/gcsStorage.js +10 -47
- package/dist/storage/adapters/historicalStorageAdapter.d.ts +2 -5
- package/dist/storage/adapters/historicalStorageAdapter.js +2 -9
- package/dist/storage/adapters/memoryStorage.d.ts +2 -5
- package/dist/storage/adapters/memoryStorage.js +2 -10
- package/dist/storage/adapters/opfsStorage.d.ts +2 -5
- package/dist/storage/adapters/opfsStorage.js +12 -54
- package/dist/storage/adapters/r2Storage.d.ts +0 -13
- package/dist/storage/adapters/r2Storage.js +18 -53
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +2 -5
- package/dist/storage/adapters/s3CompatibleStorage.js +18 -70
- package/dist/storage/baseStorage.d.ts +8 -14
- package/dist/storage/baseStorage.js +16 -40
- package/dist/storage/cow/CommitLog.d.ts +24 -0
- package/dist/storage/cow/CommitLog.js +37 -0
- package/dist/types/brainy.types.d.ts +2 -0
- package/dist/utils/paramValidation.d.ts +43 -0
- package/dist/utils/paramValidation.js +135 -22
- package/package.json +1 -1
package/dist/brainy.d.ts
CHANGED
|
@@ -1009,6 +1009,51 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
1009
1009
|
timestamp: number;
|
|
1010
1010
|
metadata?: Record<string, any>;
|
|
1011
1011
|
}>>;
|
|
1012
|
+
/**
|
|
1013
|
+
* Stream commit history (memory-efficient)
|
|
1014
|
+
*
|
|
1015
|
+
* Use this for large commit histories (1000s of snapshots) where memory
|
|
1016
|
+
* efficiency is critical. Yields commits one at a time without accumulating
|
|
1017
|
+
* them in memory.
|
|
1018
|
+
*
|
|
1019
|
+
* For small histories (< 100 commits), use getHistory() for simpler API.
|
|
1020
|
+
*
|
|
1021
|
+
* @param options - History options
|
|
1022
|
+
* @param options.limit - Maximum number of commits to stream
|
|
1023
|
+
* @param options.since - Only include commits after this timestamp
|
|
1024
|
+
* @param options.until - Only include commits before this timestamp
|
|
1025
|
+
* @param options.author - Filter by author name
|
|
1026
|
+
*
|
|
1027
|
+
* @yields Commit metadata in reverse chronological order (newest first)
|
|
1028
|
+
*
|
|
1029
|
+
* @example
|
|
1030
|
+
* ```typescript
|
|
1031
|
+
* // Stream all commits without memory accumulation
|
|
1032
|
+
* for await (const commit of brain.streamHistory({ limit: 10000 })) {
|
|
1033
|
+
* console.log(`${commit.timestamp}: ${commit.message}`)
|
|
1034
|
+
* }
|
|
1035
|
+
*
|
|
1036
|
+
* // Stream with filtering
|
|
1037
|
+
* for await (const commit of brain.streamHistory({
|
|
1038
|
+
* author: 'alice',
|
|
1039
|
+
* since: Date.now() - 86400000 // Last 24 hours
|
|
1040
|
+
* })) {
|
|
1041
|
+
* // Process commit
|
|
1042
|
+
* }
|
|
1043
|
+
* ```
|
|
1044
|
+
*/
|
|
1045
|
+
streamHistory(options?: {
|
|
1046
|
+
limit?: number;
|
|
1047
|
+
since?: number;
|
|
1048
|
+
until?: number;
|
|
1049
|
+
author?: string;
|
|
1050
|
+
}): AsyncIterableIterator<{
|
|
1051
|
+
hash: string;
|
|
1052
|
+
message: string;
|
|
1053
|
+
author: string;
|
|
1054
|
+
timestamp: number;
|
|
1055
|
+
metadata?: Record<string, any>;
|
|
1056
|
+
}>;
|
|
1012
1057
|
/**
|
|
1013
1058
|
* Get total count of nouns - O(1) operation
|
|
1014
1059
|
* @returns Promise that resolves to the total number of nouns
|
|
@@ -1019,6 +1064,50 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
1019
1064
|
* @returns Promise that resolves to the total number of verbs
|
|
1020
1065
|
*/
|
|
1021
1066
|
getVerbCount(): Promise<number>;
|
|
1067
|
+
/**
|
|
1068
|
+
* Get memory statistics and limits (v5.11.0)
|
|
1069
|
+
*
|
|
1070
|
+
* Returns detailed memory information including:
|
|
1071
|
+
* - Current heap usage
|
|
1072
|
+
* - Container memory limits (if detected)
|
|
1073
|
+
* - Query limits and how they were calculated
|
|
1074
|
+
* - Memory allocation recommendations
|
|
1075
|
+
*
|
|
1076
|
+
* Use this to debug why query limits are low or to understand
|
|
1077
|
+
* memory allocation in production environments.
|
|
1078
|
+
*
|
|
1079
|
+
* @returns Memory statistics and configuration
|
|
1080
|
+
*
|
|
1081
|
+
* @example
|
|
1082
|
+
* ```typescript
|
|
1083
|
+
* const stats = brain.getMemoryStats()
|
|
1084
|
+
* console.log(`Query limit: ${stats.limits.maxQueryLimit}`)
|
|
1085
|
+
* console.log(`Basis: ${stats.limits.basis}`)
|
|
1086
|
+
* console.log(`Free memory: ${Math.round(stats.memory.free / 1024 / 1024)}MB`)
|
|
1087
|
+
* ```
|
|
1088
|
+
*/
|
|
1089
|
+
getMemoryStats(): {
|
|
1090
|
+
memory: {
|
|
1091
|
+
heapUsed: number;
|
|
1092
|
+
heapTotal: number;
|
|
1093
|
+
external: number;
|
|
1094
|
+
rss: number;
|
|
1095
|
+
free: number;
|
|
1096
|
+
total: number;
|
|
1097
|
+
containerLimit: number | null;
|
|
1098
|
+
};
|
|
1099
|
+
limits: {
|
|
1100
|
+
maxQueryLimit: number;
|
|
1101
|
+
maxQueryLength: number;
|
|
1102
|
+
maxVectorDimensions: number;
|
|
1103
|
+
basis: 'override' | 'reservedMemory' | 'containerMemory' | 'freeMemory';
|
|
1104
|
+
};
|
|
1105
|
+
config: {
|
|
1106
|
+
maxQueryLimit?: number;
|
|
1107
|
+
reservedQueryMemory?: number;
|
|
1108
|
+
};
|
|
1109
|
+
recommendations?: string[];
|
|
1110
|
+
};
|
|
1022
1111
|
/**
|
|
1023
1112
|
* Neural API - Advanced AI operations
|
|
1024
1113
|
*/
|
package/dist/brainy.js
CHANGED
|
@@ -25,6 +25,7 @@ import { NULL_HASH } from './storage/cow/constants.js';
|
|
|
25
25
|
import { createPipeline } from './streaming/pipeline.js';
|
|
26
26
|
import { configureLogger, LogLevel } from './utils/logger.js';
|
|
27
27
|
import { TransactionManager } from './transaction/TransactionManager.js';
|
|
28
|
+
import { ValidationConfig } from './utils/paramValidation.js';
|
|
28
29
|
import { SaveNounMetadataOperation, SaveNounOperation, AddToTypeAwareHNSWOperation, AddToHNSWOperation, AddToMetadataIndexOperation, SaveVerbMetadataOperation, SaveVerbOperation, AddToGraphIndexOperation, RemoveFromHNSWOperation, RemoveFromTypeAwareHNSWOperation, RemoveFromMetadataIndexOperation, RemoveFromGraphIndexOperation, UpdateNounMetadataOperation, DeleteNounMetadataOperation, DeleteVerbMetadataOperation } from './transaction/operations/index.js';
|
|
29
30
|
import { DistributedCoordinator, ShardManager, CacheSync, ReadWriteSeparation } from './distributed/index.js';
|
|
30
31
|
import { NounType } from './types/graphTypes.js';
|
|
@@ -45,6 +46,14 @@ export class Brainy {
|
|
|
45
46
|
this.lazyRebuildPromise = null;
|
|
46
47
|
// Normalize configuration with defaults
|
|
47
48
|
this.config = this.normalizeConfig(config);
|
|
49
|
+
// Configure memory limits (v5.11.0)
|
|
50
|
+
// This must happen early, before any validation occurs
|
|
51
|
+
if (this.config.maxQueryLimit !== undefined || this.config.reservedQueryMemory !== undefined) {
|
|
52
|
+
ValidationConfig.reconfigure({
|
|
53
|
+
maxQueryLimit: this.config.maxQueryLimit,
|
|
54
|
+
reservedQueryMemory: this.config.reservedQueryMemory
|
|
55
|
+
});
|
|
56
|
+
}
|
|
48
57
|
// Setup core components
|
|
49
58
|
this.distance = cosineDistance;
|
|
50
59
|
this.embedder = this.setupEmbedder();
|
|
@@ -2688,6 +2697,67 @@ export class Brainy {
|
|
|
2688
2697
|
}));
|
|
2689
2698
|
});
|
|
2690
2699
|
}
|
|
2700
|
+
/**
|
|
2701
|
+
* Stream commit history (memory-efficient)
|
|
2702
|
+
*
|
|
2703
|
+
* Use this for large commit histories (1000s of snapshots) where memory
|
|
2704
|
+
* efficiency is critical. Yields commits one at a time without accumulating
|
|
2705
|
+
* them in memory.
|
|
2706
|
+
*
|
|
2707
|
+
* For small histories (< 100 commits), use getHistory() for simpler API.
|
|
2708
|
+
*
|
|
2709
|
+
* @param options - History options
|
|
2710
|
+
* @param options.limit - Maximum number of commits to stream
|
|
2711
|
+
* @param options.since - Only include commits after this timestamp
|
|
2712
|
+
* @param options.until - Only include commits before this timestamp
|
|
2713
|
+
* @param options.author - Filter by author name
|
|
2714
|
+
*
|
|
2715
|
+
* @yields Commit metadata in reverse chronological order (newest first)
|
|
2716
|
+
*
|
|
2717
|
+
* @example
|
|
2718
|
+
* ```typescript
|
|
2719
|
+
* // Stream all commits without memory accumulation
|
|
2720
|
+
* for await (const commit of brain.streamHistory({ limit: 10000 })) {
|
|
2721
|
+
* console.log(`${commit.timestamp}: ${commit.message}`)
|
|
2722
|
+
* }
|
|
2723
|
+
*
|
|
2724
|
+
* // Stream with filtering
|
|
2725
|
+
* for await (const commit of brain.streamHistory({
|
|
2726
|
+
* author: 'alice',
|
|
2727
|
+
* since: Date.now() - 86400000 // Last 24 hours
|
|
2728
|
+
* })) {
|
|
2729
|
+
* // Process commit
|
|
2730
|
+
* }
|
|
2731
|
+
* ```
|
|
2732
|
+
*/
|
|
2733
|
+
async *streamHistory(options) {
|
|
2734
|
+
await this.ensureInitialized();
|
|
2735
|
+
if (!('commitLog' in this.storage) || !('refManager' in this.storage)) {
|
|
2736
|
+
throw new Error('History streaming requires COW-enabled storage (v5.0.0+)');
|
|
2737
|
+
}
|
|
2738
|
+
const commitLog = this.storage.commitLog;
|
|
2739
|
+
const currentBranch = await this.getCurrentBranch();
|
|
2740
|
+
const blobStorage = this.storage.blobStorage;
|
|
2741
|
+
// Stream commits from CommitLog
|
|
2742
|
+
for await (const commit of commitLog.streamHistory(currentBranch, {
|
|
2743
|
+
maxCount: options?.limit,
|
|
2744
|
+
since: options?.since,
|
|
2745
|
+
until: options?.until
|
|
2746
|
+
})) {
|
|
2747
|
+
// Filter by author if specified
|
|
2748
|
+
if (options?.author && commit.author !== options.author) {
|
|
2749
|
+
continue;
|
|
2750
|
+
}
|
|
2751
|
+
// Map to expected format (compute hash for commit)
|
|
2752
|
+
yield {
|
|
2753
|
+
hash: blobStorage.constructor.hash(Buffer.from(JSON.stringify(commit))),
|
|
2754
|
+
message: commit.message,
|
|
2755
|
+
author: commit.author,
|
|
2756
|
+
timestamp: commit.timestamp,
|
|
2757
|
+
metadata: commit.metadata
|
|
2758
|
+
};
|
|
2759
|
+
}
|
|
2760
|
+
}
|
|
2691
2761
|
/**
|
|
2692
2762
|
* Get total count of nouns - O(1) operation
|
|
2693
2763
|
* @returns Promise that resolves to the total number of nouns
|
|
@@ -2704,6 +2774,86 @@ export class Brainy {
|
|
|
2704
2774
|
await this.ensureInitialized();
|
|
2705
2775
|
return this.storage.getVerbCount();
|
|
2706
2776
|
}
|
|
2777
|
+
/**
|
|
2778
|
+
* Get memory statistics and limits (v5.11.0)
|
|
2779
|
+
*
|
|
2780
|
+
* Returns detailed memory information including:
|
|
2781
|
+
* - Current heap usage
|
|
2782
|
+
* - Container memory limits (if detected)
|
|
2783
|
+
* - Query limits and how they were calculated
|
|
2784
|
+
* - Memory allocation recommendations
|
|
2785
|
+
*
|
|
2786
|
+
* Use this to debug why query limits are low or to understand
|
|
2787
|
+
* memory allocation in production environments.
|
|
2788
|
+
*
|
|
2789
|
+
* @returns Memory statistics and configuration
|
|
2790
|
+
*
|
|
2791
|
+
* @example
|
|
2792
|
+
* ```typescript
|
|
2793
|
+
* const stats = brain.getMemoryStats()
|
|
2794
|
+
* console.log(`Query limit: ${stats.limits.maxQueryLimit}`)
|
|
2795
|
+
* console.log(`Basis: ${stats.limits.basis}`)
|
|
2796
|
+
* console.log(`Free memory: ${Math.round(stats.memory.free / 1024 / 1024)}MB`)
|
|
2797
|
+
* ```
|
|
2798
|
+
*/
|
|
2799
|
+
getMemoryStats() {
|
|
2800
|
+
const config = ValidationConfig.getInstance();
|
|
2801
|
+
const heapStats = process.memoryUsage ? process.memoryUsage() : {
|
|
2802
|
+
heapUsed: 0,
|
|
2803
|
+
heapTotal: 0,
|
|
2804
|
+
external: 0,
|
|
2805
|
+
rss: 0
|
|
2806
|
+
};
|
|
2807
|
+
// Get system memory info
|
|
2808
|
+
let freeMemory = 0;
|
|
2809
|
+
let totalMemory = 0;
|
|
2810
|
+
if (typeof window === 'undefined') {
|
|
2811
|
+
try {
|
|
2812
|
+
const os = require('node:os');
|
|
2813
|
+
freeMemory = os.freemem();
|
|
2814
|
+
totalMemory = os.totalmem();
|
|
2815
|
+
}
|
|
2816
|
+
catch (e) {
|
|
2817
|
+
// OS module not available
|
|
2818
|
+
}
|
|
2819
|
+
}
|
|
2820
|
+
const stats = {
|
|
2821
|
+
memory: {
|
|
2822
|
+
heapUsed: heapStats.heapUsed,
|
|
2823
|
+
heapTotal: heapStats.heapTotal,
|
|
2824
|
+
external: heapStats.external,
|
|
2825
|
+
rss: heapStats.rss,
|
|
2826
|
+
free: freeMemory,
|
|
2827
|
+
total: totalMemory,
|
|
2828
|
+
containerLimit: config.detectedContainerLimit
|
|
2829
|
+
},
|
|
2830
|
+
limits: {
|
|
2831
|
+
maxQueryLimit: config.maxLimit,
|
|
2832
|
+
maxQueryLength: config.maxQueryLength,
|
|
2833
|
+
maxVectorDimensions: config.maxVectorDimensions,
|
|
2834
|
+
basis: config.limitBasis
|
|
2835
|
+
},
|
|
2836
|
+
config: {
|
|
2837
|
+
maxQueryLimit: this.config.maxQueryLimit,
|
|
2838
|
+
reservedQueryMemory: this.config.reservedQueryMemory
|
|
2839
|
+
},
|
|
2840
|
+
recommendations: []
|
|
2841
|
+
};
|
|
2842
|
+
// Generate recommendations based on stats
|
|
2843
|
+
if (stats.limits.basis === 'freeMemory' && stats.memory.containerLimit) {
|
|
2844
|
+
stats.recommendations.push(`Container detected (${Math.round(stats.memory.containerLimit / 1024 / 1024)}MB) but limits based on free memory. ` +
|
|
2845
|
+
`Consider setting reservedQueryMemory config option for better limits.`);
|
|
2846
|
+
}
|
|
2847
|
+
if (stats.limits.maxQueryLimit < 5000 && stats.memory.containerLimit && stats.memory.containerLimit > 2 * 1024 * 1024 * 1024) {
|
|
2848
|
+
stats.recommendations.push(`Query limit is low (${stats.limits.maxQueryLimit}) despite ${Math.round(stats.memory.containerLimit / 1024 / 1024 / 1024)}GB container. ` +
|
|
2849
|
+
`Consider: new Brainy({ reservedQueryMemory: 1073741824 }) to reserve 1GB for queries.`);
|
|
2850
|
+
}
|
|
2851
|
+
if (stats.limits.basis === 'override') {
|
|
2852
|
+
stats.recommendations.push(`Using explicit maxQueryLimit override (${stats.limits.maxQueryLimit}). ` +
|
|
2853
|
+
`Auto-detection bypassed.`);
|
|
2854
|
+
}
|
|
2855
|
+
return stats;
|
|
2856
|
+
}
|
|
2707
2857
|
// ============= SUB-APIS =============
|
|
2708
2858
|
/**
|
|
2709
2859
|
* Neural API - Advanced AI operations
|
|
@@ -3993,7 +4143,10 @@ export class Brainy {
|
|
|
3993
4143
|
disableMetrics: config?.disableMetrics ?? false,
|
|
3994
4144
|
disableAutoOptimize: config?.disableAutoOptimize ?? false,
|
|
3995
4145
|
batchWrites: config?.batchWrites ?? true,
|
|
3996
|
-
maxConcurrentOperations: config?.maxConcurrentOperations ?? 10
|
|
4146
|
+
maxConcurrentOperations: config?.maxConcurrentOperations ?? 10,
|
|
4147
|
+
// Memory management options (v5.11.0)
|
|
4148
|
+
maxQueryLimit: config?.maxQueryLimit ?? undefined,
|
|
4149
|
+
reservedQueryMemory: config?.reservedQueryMemory ?? undefined
|
|
3997
4150
|
};
|
|
3998
4151
|
}
|
|
3999
4152
|
/**
|
|
@@ -246,13 +246,10 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
246
246
|
* @returns true if marker blob exists, false otherwise
|
|
247
247
|
* @protected
|
|
248
248
|
*/
|
|
249
|
-
protected checkClearMarker(): Promise<boolean>;
|
|
250
249
|
/**
|
|
251
|
-
*
|
|
252
|
-
*
|
|
253
|
-
* @protected
|
|
250
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
251
|
+
* COW is now always enabled - marker files are no longer used
|
|
254
252
|
*/
|
|
255
|
-
protected createClearMarker(): Promise<void>;
|
|
256
253
|
/**
|
|
257
254
|
* Save statistics data to storage
|
|
258
255
|
*/
|
|
@@ -859,18 +859,11 @@ export class AzureBlobStorage extends BaseStorage {
|
|
|
859
859
|
await blockBlobClient.delete();
|
|
860
860
|
}
|
|
861
861
|
}
|
|
862
|
-
//
|
|
863
|
-
//
|
|
864
|
-
// Otherwise initializeCOW() will auto-recreate initial commit on next operation
|
|
862
|
+
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
|
863
|
+
// COW will re-initialize automatically on next use
|
|
865
864
|
this.refManager = undefined;
|
|
866
865
|
this.blobStorage = undefined;
|
|
867
866
|
this.commitLog = undefined;
|
|
868
|
-
this.cowEnabled = false;
|
|
869
|
-
// v5.10.4: Create persistent marker blob (CRITICAL FIX)
|
|
870
|
-
// Bug: cowEnabled = false only affects current instance, not future instances
|
|
871
|
-
// Fix: Create marker blob that persists across instance restarts
|
|
872
|
-
// When new instance calls initializeCOW(), it checks for this marker
|
|
873
|
-
await this.createClearMarker();
|
|
874
867
|
// Clear caches
|
|
875
868
|
this.nounCacheManager.clear();
|
|
876
869
|
this.verbCacheManager.clear();
|
|
@@ -919,39 +912,10 @@ export class AzureBlobStorage extends BaseStorage {
|
|
|
919
912
|
* @returns true if marker blob exists, false otherwise
|
|
920
913
|
* @protected
|
|
921
914
|
*/
|
|
922
|
-
async checkClearMarker() {
|
|
923
|
-
await this.ensureInitialized();
|
|
924
|
-
try {
|
|
925
|
-
const markerPath = `${this.systemPrefix}cow-disabled`;
|
|
926
|
-
const blockBlobClient = this.containerClient.getBlockBlobClient(markerPath);
|
|
927
|
-
const exists = await blockBlobClient.exists();
|
|
928
|
-
return exists;
|
|
929
|
-
}
|
|
930
|
-
catch (error) {
|
|
931
|
-
this.logger.warn('AzureBlobStorage.checkClearMarker: Error checking marker', error);
|
|
932
|
-
return false;
|
|
933
|
-
}
|
|
934
|
-
}
|
|
935
915
|
/**
|
|
936
|
-
*
|
|
937
|
-
*
|
|
938
|
-
* @protected
|
|
916
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
917
|
+
* COW is now always enabled - marker files are no longer used
|
|
939
918
|
*/
|
|
940
|
-
async createClearMarker() {
|
|
941
|
-
await this.ensureInitialized();
|
|
942
|
-
try {
|
|
943
|
-
const markerPath = `${this.systemPrefix}cow-disabled`;
|
|
944
|
-
const blockBlobClient = this.containerClient.getBlockBlobClient(markerPath);
|
|
945
|
-
// Create empty marker blob
|
|
946
|
-
await blockBlobClient.upload(Buffer.from(''), 0, {
|
|
947
|
-
blobHTTPHeaders: { blobContentType: 'text/plain' }
|
|
948
|
-
});
|
|
949
|
-
}
|
|
950
|
-
catch (error) {
|
|
951
|
-
this.logger.error('AzureBlobStorage.createClearMarker: Failed to create marker blob', error);
|
|
952
|
-
// Don't throw - marker creation failure shouldn't break clear()
|
|
953
|
-
}
|
|
954
|
-
}
|
|
955
919
|
/**
|
|
956
920
|
* Save statistics data to storage
|
|
957
921
|
*/
|
|
@@ -177,13 +177,10 @@ export declare class FileSystemStorage extends BaseStorage {
|
|
|
177
177
|
* @returns true if marker file exists, false otherwise
|
|
178
178
|
* @protected
|
|
179
179
|
*/
|
|
180
|
-
protected checkClearMarker(): Promise<boolean>;
|
|
181
180
|
/**
|
|
182
|
-
*
|
|
183
|
-
*
|
|
184
|
-
* @protected
|
|
181
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
182
|
+
* COW is now always enabled - marker files are no longer used
|
|
185
183
|
*/
|
|
186
|
-
protected createClearMarker(): Promise<void>;
|
|
187
184
|
/**
|
|
188
185
|
* Get information about storage usage and capacity
|
|
189
186
|
*/
|
|
@@ -878,18 +878,11 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
878
878
|
if (await this.directoryExists(cowDir)) {
|
|
879
879
|
// Delete the entire _cow/ directory (not just contents)
|
|
880
880
|
await fs.promises.rm(cowDir, { recursive: true, force: true });
|
|
881
|
-
//
|
|
882
|
-
//
|
|
883
|
-
// Otherwise initializeCOW() will auto-recreate initial commit on next operation
|
|
881
|
+
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
|
882
|
+
// COW will re-initialize automatically on next use
|
|
884
883
|
this.refManager = undefined;
|
|
885
884
|
this.blobStorage = undefined;
|
|
886
885
|
this.commitLog = undefined;
|
|
887
|
-
this.cowEnabled = false;
|
|
888
|
-
// v5.10.4: Create persistent marker file (CRITICAL FIX)
|
|
889
|
-
// Bug: cowEnabled = false only affects current instance, not future instances
|
|
890
|
-
// Fix: Create marker file that persists across instance restarts
|
|
891
|
-
// When new instance calls initializeCOW(), it checks for this marker
|
|
892
|
-
await this.createClearMarker();
|
|
893
886
|
}
|
|
894
887
|
// Clear the statistics cache
|
|
895
888
|
this.statisticsCache = null;
|
|
@@ -923,41 +916,10 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
923
916
|
* @returns true if marker file exists, false otherwise
|
|
924
917
|
* @protected
|
|
925
918
|
*/
|
|
926
|
-
async checkClearMarker() {
|
|
927
|
-
// Check if fs module is available
|
|
928
|
-
if (!fs || !fs.promises) {
|
|
929
|
-
return false;
|
|
930
|
-
}
|
|
931
|
-
try {
|
|
932
|
-
const markerPath = path.join(this.systemDir, 'cow-disabled');
|
|
933
|
-
await fs.promises.access(markerPath, fs.constants.F_OK);
|
|
934
|
-
return true; // Marker exists
|
|
935
|
-
}
|
|
936
|
-
catch (error) {
|
|
937
|
-
return false; // Marker doesn't exist (ENOENT) or can't be accessed
|
|
938
|
-
}
|
|
939
|
-
}
|
|
940
919
|
/**
|
|
941
|
-
*
|
|
942
|
-
*
|
|
943
|
-
* @protected
|
|
920
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
921
|
+
* COW is now always enabled - marker files are no longer used
|
|
944
922
|
*/
|
|
945
|
-
async createClearMarker() {
|
|
946
|
-
// Check if fs module is available
|
|
947
|
-
if (!fs || !fs.promises) {
|
|
948
|
-
console.warn('FileSystemStorage.createClearMarker: fs module not available, skipping marker creation');
|
|
949
|
-
return;
|
|
950
|
-
}
|
|
951
|
-
try {
|
|
952
|
-
const markerPath = path.join(this.systemDir, 'cow-disabled');
|
|
953
|
-
// Create empty marker file
|
|
954
|
-
await fs.promises.writeFile(markerPath, '', 'utf8');
|
|
955
|
-
}
|
|
956
|
-
catch (error) {
|
|
957
|
-
console.error('FileSystemStorage.createClearMarker: Failed to create marker file', error);
|
|
958
|
-
// Don't throw - marker creation failure shouldn't break clear()
|
|
959
|
-
}
|
|
960
|
-
}
|
|
961
923
|
/**
|
|
962
924
|
* Get information about storage usage and capacity
|
|
963
925
|
*/
|
|
@@ -234,13 +234,10 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
234
234
|
* @returns true if marker object exists, false otherwise
|
|
235
235
|
* @protected
|
|
236
236
|
*/
|
|
237
|
-
protected checkClearMarker(): Promise<boolean>;
|
|
238
237
|
/**
|
|
239
|
-
*
|
|
240
|
-
*
|
|
241
|
-
* @protected
|
|
238
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
239
|
+
* COW is now always enabled - marker files are no longer used
|
|
242
240
|
*/
|
|
243
|
-
protected createClearMarker(): Promise<void>;
|
|
244
241
|
/**
|
|
245
242
|
* Save statistics data to storage
|
|
246
243
|
*/
|
|
@@ -772,28 +772,18 @@ export class GcsStorage extends BaseStorage {
|
|
|
772
772
|
await file.delete();
|
|
773
773
|
}
|
|
774
774
|
};
|
|
775
|
-
// Clear
|
|
776
|
-
|
|
777
|
-
await deleteObjectsWithPrefix(
|
|
778
|
-
|
|
779
|
-
await deleteObjectsWithPrefix(this.verbMetadataPrefix);
|
|
780
|
-
await deleteObjectsWithPrefix(this.systemPrefix);
|
|
781
|
-
// v5.6.1: Clear COW (copy-on-write) version control data
|
|
782
|
-
// This includes all git-like versioning data (commits, trees, blobs, refs)
|
|
783
|
-
// Must be deleted to fully clear all data including version history
|
|
775
|
+
// v5.11.0: Clear ALL data using correct paths
|
|
776
|
+
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
|
777
|
+
await deleteObjectsWithPrefix('branches/');
|
|
778
|
+
// Delete COW version control data
|
|
784
779
|
await deleteObjectsWithPrefix('_cow/');
|
|
785
|
-
//
|
|
786
|
-
|
|
787
|
-
//
|
|
780
|
+
// Delete system metadata
|
|
781
|
+
await deleteObjectsWithPrefix('_system/');
|
|
782
|
+
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
|
783
|
+
// COW will re-initialize automatically on next use
|
|
788
784
|
this.refManager = undefined;
|
|
789
785
|
this.blobStorage = undefined;
|
|
790
786
|
this.commitLog = undefined;
|
|
791
|
-
this.cowEnabled = false;
|
|
792
|
-
// v5.10.4: Create persistent marker object (CRITICAL FIX)
|
|
793
|
-
// Bug: cowEnabled = false only affects current instance, not future instances
|
|
794
|
-
// Fix: Create marker object that persists across instance restarts
|
|
795
|
-
// When new instance calls initializeCOW(), it checks for this marker
|
|
796
|
-
await this.createClearMarker();
|
|
797
787
|
// Clear caches
|
|
798
788
|
this.nounCacheManager.clear();
|
|
799
789
|
this.verbCacheManager.clear();
|
|
@@ -845,37 +835,10 @@ export class GcsStorage extends BaseStorage {
|
|
|
845
835
|
* @returns true if marker object exists, false otherwise
|
|
846
836
|
* @protected
|
|
847
837
|
*/
|
|
848
|
-
async checkClearMarker() {
|
|
849
|
-
await this.ensureInitialized();
|
|
850
|
-
try {
|
|
851
|
-
const markerPath = `${this.systemPrefix}cow-disabled`;
|
|
852
|
-
const file = this.bucket.file(markerPath);
|
|
853
|
-
const [exists] = await file.exists();
|
|
854
|
-
return exists;
|
|
855
|
-
}
|
|
856
|
-
catch (error) {
|
|
857
|
-
this.logger.warn('GCSStorage.checkClearMarker: Error checking marker', error);
|
|
858
|
-
return false;
|
|
859
|
-
}
|
|
860
|
-
}
|
|
861
838
|
/**
|
|
862
|
-
*
|
|
863
|
-
*
|
|
864
|
-
* @protected
|
|
839
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
840
|
+
* COW is now always enabled - marker files are no longer used
|
|
865
841
|
*/
|
|
866
|
-
async createClearMarker() {
|
|
867
|
-
await this.ensureInitialized();
|
|
868
|
-
try {
|
|
869
|
-
const markerPath = `${this.systemPrefix}cow-disabled`;
|
|
870
|
-
const file = this.bucket.file(markerPath);
|
|
871
|
-
// Create empty marker object
|
|
872
|
-
await file.save('', { contentType: 'text/plain' });
|
|
873
|
-
}
|
|
874
|
-
catch (error) {
|
|
875
|
-
this.logger.error('GCSStorage.createClearMarker: Failed to create marker object', error);
|
|
876
|
-
// Don't throw - marker creation failure shouldn't break clear()
|
|
877
|
-
}
|
|
878
|
-
}
|
|
879
842
|
/**
|
|
880
843
|
* Save statistics data to storage
|
|
881
844
|
*/
|
|
@@ -107,13 +107,10 @@ export declare class HistoricalStorageAdapter extends BaseStorage {
|
|
|
107
107
|
* @returns Always false (read-only adapter doesn't manage COW state)
|
|
108
108
|
* @protected
|
|
109
109
|
*/
|
|
110
|
-
protected checkClearMarker(): Promise<boolean>;
|
|
111
110
|
/**
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
* @protected
|
|
111
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
112
|
+
* COW is now always enabled - marker files are no longer used
|
|
115
113
|
*/
|
|
116
|
-
protected createClearMarker(): Promise<void>;
|
|
117
114
|
/**
|
|
118
115
|
* WRITE BLOCKED: Historical storage is read-only
|
|
119
116
|
*/
|
|
@@ -232,17 +232,10 @@ export class HistoricalStorageAdapter extends BaseStorage {
|
|
|
232
232
|
* @returns Always false (read-only adapter doesn't manage COW state)
|
|
233
233
|
* @protected
|
|
234
234
|
*/
|
|
235
|
-
async checkClearMarker() {
|
|
236
|
-
return false; // Read-only adapter - COW state managed by underlying storage
|
|
237
|
-
}
|
|
238
235
|
/**
|
|
239
|
-
*
|
|
240
|
-
*
|
|
241
|
-
* @protected
|
|
236
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
237
|
+
* COW is now always enabled - marker files are no longer used
|
|
242
238
|
*/
|
|
243
|
-
async createClearMarker() {
|
|
244
|
-
// No-op: HistoricalStorageAdapter is read-only, doesn't create markers
|
|
245
|
-
}
|
|
246
239
|
// ============= Override Write Methods (Read-Only) =============
|
|
247
240
|
/**
|
|
248
241
|
* WRITE BLOCKED: Historical storage is read-only
|
|
@@ -85,13 +85,10 @@ export declare class MemoryStorage extends BaseStorage {
|
|
|
85
85
|
* @returns Always false (marker doesn't persist in memory)
|
|
86
86
|
* @protected
|
|
87
87
|
*/
|
|
88
|
-
protected checkClearMarker(): Promise<boolean>;
|
|
89
88
|
/**
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
* @protected
|
|
89
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
90
|
+
* COW is now always enabled - marker files are no longer used
|
|
93
91
|
*/
|
|
94
|
-
protected createClearMarker(): Promise<void>;
|
|
95
92
|
/**
|
|
96
93
|
* Save statistics data to storage
|
|
97
94
|
* @param statistics The statistics data to save
|
|
@@ -164,18 +164,10 @@ export class MemoryStorage extends BaseStorage {
|
|
|
164
164
|
* @returns Always false (marker doesn't persist in memory)
|
|
165
165
|
* @protected
|
|
166
166
|
*/
|
|
167
|
-
async checkClearMarker() {
|
|
168
|
-
return false; // MemoryStorage doesn't persist - marker doesn't survive restart
|
|
169
|
-
}
|
|
170
167
|
/**
|
|
171
|
-
*
|
|
172
|
-
*
|
|
173
|
-
* @protected
|
|
168
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
169
|
+
* COW is now always enabled - marker files are no longer used
|
|
174
170
|
*/
|
|
175
|
-
async createClearMarker() {
|
|
176
|
-
// No-op: MemoryStorage doesn't persist, so marker is not needed
|
|
177
|
-
// clear() in memory already resets all state, no marker survives restart
|
|
178
|
-
}
|
|
179
171
|
/**
|
|
180
172
|
* Save statistics data to storage
|
|
181
173
|
* @param statistics The statistics data to save
|
|
@@ -100,13 +100,10 @@ export declare class OPFSStorage extends BaseStorage {
|
|
|
100
100
|
* @returns true if marker file exists, false otherwise
|
|
101
101
|
* @protected
|
|
102
102
|
*/
|
|
103
|
-
protected checkClearMarker(): Promise<boolean>;
|
|
104
103
|
/**
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
* @protected
|
|
104
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
105
|
+
* COW is now always enabled - marker files are no longer used
|
|
108
106
|
*/
|
|
109
|
-
protected createClearMarker(): Promise<void>;
|
|
110
107
|
private quotaWarningThreshold;
|
|
111
108
|
private quotaCriticalThreshold;
|
|
112
109
|
private lastQuotaCheck;
|